Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2025-10-30 18:46:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use less memory to build translation files (#6070)

Browse Source 
* Change languages build to use less memory

Add graceful-fs so no ned to worry about fs limits

* Specify encoding for fs.readFile

Use eachLimit since graceful-fs handles that now
This commit is contained in:
Peter Jaszkowiak
2017-11-16 15:43:52 -07:00
committed by Barış Soner Uşaklı
parent f5385e38bf
commit c47c47f7e3
17 changed files with 142 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
package.default.json
View File

@@ -40,6 +40,7 @@
"express": "^4.16.2",
"express-session": "^1.15.6",
"express-useragent": "1.0.8",
"graceful-fs": "^4.1.11",
"html-to-text": "3.3.0",
"ipaddr.js": "^1.5.4",
"jimp": "0.2.28",

3
src/admin/search.js
View File

@@ -63,12 +63,11 @@ var fallbackCacheInProgress = {};
var fallbackCache = {};
function initFallback(namespace, callback) {
fs.readFile(path.resolve(nconf.get('views_dir'), namespace + '.tpl'), function (err, file) {
fs.readFile(path.resolve(nconf.get('views_dir'), namespace + '.tpl'), 'utf8', function (err, template) {
if (err) {
return callback(err);
}
var template = file.toString();
var title = nsToTitle(namespace);
var translations = sanitize(template);

6
src/controllers/admin/settings.js
View File

@@ -45,16 +45,16 @@ function renderEmail(req, res, next) {
async.waterfall([
function (next) {
fs.readFile(email, next);
fs.readFile(email, 'utf8', next);
},
function (original, next) {
var text = meta.config['email:custom:' + path] ? meta.config['email:custom:' + path] : original.toString();
var text = meta.config['email:custom:' + path] ? meta.config['email:custom:' + path] : original;
next(null, {
path: path,
fullpath: email,
text: text,
original: original.toString(),
original: original,
});
},
], next);

2
src/controllers/admin/themes.js
View File

@@ -23,7 +23,7 @@ themesController.get = function (req, res, next) {
return next(Error('invalid-data'));
}
fs.readFile(themeConfigPath, next);
fs.readFile(themeConfigPath, 'utf8', next);
},
function (themeConfig, next) {
try {

3
src/file.js
View File

@@ -7,9 +7,12 @@ var winston = require('winston');
var jimp = require('jimp');
var mkdirp = require('mkdirp');
var mime = require('mime');
var graceful = require('graceful-fs');
var utils = require('./utils');
graceful.gracefulify(fs);
var file = module.exports;
/**

4
src/image.js
View File

@@ -120,9 +120,7 @@ image.size = function (path, callback) {
};
image.convertImageToBase64 = function (path, callback) {
fs.readFile(path, function (err, data) {
callback(err, data ? data.toString('base64') : null);
});
fs.readFile(path, 'base64', callback);
};
image.mimeFromBase64 = function (imageData) {

10
src/install.js
View File

@@ -371,7 +371,7 @@ function createCategories(next) {
process.stdout.write('No categories found, populating instance with default categories\n');
fs.readFile(path.join(__dirname, '../', 'install/data/categories.json'), function (err, default_categories) {
fs.readFile(path.join(__dirname, '../', 'install/data/categories.json'), 'utf8', function (err, default_categories) {
if (err) {
return next(err);
}
@@ -402,7 +402,7 @@ function createWelcomePost(next) {
async.parallel([
function (next) {
fs.readFile(path.join(__dirname, '../', 'install/data/welcome.md'), next);
fs.readFile(path.join(__dirname, '../', 'install/data/welcome.md'), 'utf8', next);
},
function (next) {
db.getObjectField('global', 'topicCount', next);
@@ -421,7 +421,7 @@ function createWelcomePost(next) {
uid: 1,
cid: 2,
title: 'Welcome to your NodeBB!',
content: content.toString(),
content: content,
}, next);
} else {
next();
@@ -473,7 +473,7 @@ function setCopyrightWidget(next) {
var db = require('./database');
async.parallel({
footerJSON: function (next) {
fs.readFile(path.join(__dirname, '../', 'install/data/footer.json'), next);
fs.readFile(path.join(__dirname, '../', 'install/data/footer.json'), 'utf8', next);
},
footer: function (next) {
db.getObjectField('widgets:global', 'footer', next);
@@ -484,7 +484,7 @@ function setCopyrightWidget(next) {
}
if (!results.footer && results.footerJSON) {
db.setObjectField('widgets:global', 'footer', results.footerJSON.toString(), next);
db.setObjectField('widgets:global', 'footer', results.footerJSON, next);
} else {
next();
}

8
src/languages.js
View File

@@ -29,7 +29,7 @@ Languages.listCodes = function (callback) {
return callback(null, codeCache);
}
fs.readFile(path.join(languagesPath, 'metadata.json'), function (err, buffer) {
fs.readFile(path.join(languagesPath, 'metadata.json'), 'utf8', function (err, file) {
if (err && err.code === 'ENOENT') {
return callback(null, []);
}
@@ -39,7 +39,7 @@ Languages.listCodes = function (callback) {
var parsed;
try {
parsed = JSON.parse(buffer.toString());
parsed = JSON.parse(file);
} catch (e) {
return callback(e);
}
@@ -64,7 +64,7 @@ Languages.list = function (callback) {
async.map(codes, function (folder, next) {
var configPath = path.join(languagesPath, folder, 'language.json');
fs.readFile(configPath, function (err, buffer) {
fs.readFile(configPath, 'utf8', function (err, file) {
if (err && err.code === 'ENOENT') {
return next();
}
@@ -72,7 +72,7 @@ Languages.list = function (callback) {
return next(err);
}
try {
var lang = JSON.parse(buffer.toString());
var lang = JSON.parse(file);
next(null, lang);
} catch (e) {
next(e);

8
src/meta/cacheBuster.js
View File

@@ -31,18 +31,18 @@ exports.read = function read(callback) {
return callback(null, cached);
}
fs.readFile(filePath, function (err, buffer) {
fs.readFile(filePath, 'utf8', function (err, buster) {
if (err) {
winston.warn('[cache-buster] could not read cache buster', err);
return callback(null, generate());
}
if (!buffer || buffer.toString().length !== 11) {
winston.warn('[cache-buster] cache buster string invalid: expected /[a-z0-9]{11}/, got `' + buffer + '`');
if (!buster || buster.length !== 11) {
winston.warn('[cache-buster] cache buster string invalid: expected /[a-z0-9]{11}/, got `' + buster + '`');
return callback(null, generate());
}
cached = buffer.toString();
cached = buster;
callback(null, cached);
});
};

10
src/meta/js.js
View File

@@ -106,7 +106,7 @@ function minifyModules(modules, fork, callback) {
return prev;
}, []);
async.eachLimit(moduleDirs, 1000, mkdirp, function (err) {
async.each(moduleDirs, mkdirp, function (err) {
if (err) {
return callback(err);
}
@@ -126,7 +126,7 @@ function minifyModules(modules, fork, callback) {
minifier.js.minifyBatch(filtered.minify, fork, cb);
},
function (cb) {
async.eachLimit(filtered.skip, 500, function (mod, next) {
async.each(filtered.skip, function (mod, next) {
linkIfLinux(mod.srcPath, mod.destPath, next);
}, cb);
},
@@ -137,7 +137,7 @@ function minifyModules(modules, fork, callback) {
function linkModules(callback) {
var modules = JS.scripts.modules;
async.eachLimit(Object.keys(modules), 1000, function (relPath, next) {
async.each(Object.keys(modules), function (relPath, next) {
var srcPath = path.join(__dirname, '../../', modules[relPath]);
var destPath = path.join(__dirname, '../../build/public/src/modules', relPath);
@@ -183,7 +183,7 @@ function getModuleList(callback) {
modules = modules.concat(coreDirs);
var moduleFiles = [];
async.eachLimit(modules, 1000, function (module, next) {
async.each(modules, function (module, next) {
var srcPath = module.srcPath;
var destPath = module.destPath;
@@ -255,7 +255,7 @@ JS.linkStatics = function (callback) {
if (err) {
return callback(err);
}
async.eachLimit(Object.keys(plugins.staticDirs), 1000, function (mappedPath, next) {
async.each(Object.keys(plugins.staticDirs), function (mappedPath, next) {
var sourceDir = plugins.staticDirs[mappedPath];
var destDir = path.join(__dirname, '../../build/public/plugins', mappedPath);

80
src/meta/languages.js
View File

@@ -13,7 +13,7 @@ var Plugins = require('../plugins');
var buildLanguagesPath = path.join(__dirname, '../../build/public/language');
var coreLanguagesPath = path.join(__dirname, '../../public/language');
function getTranslationTree(callback) {
function getTranslationMetadata(callback) {
async.waterfall([
// generate list of languages and namespaces
function (next) {
@@ -49,42 +49,51 @@ function getTranslationTree(callback) {
// save a list of languages to `${buildLanguagesPath}/metadata.json`
// avoids readdirs later on
function (ref, next) {
async.waterfall([
async.series([
function (next) {
mkdirp(buildLanguagesPath, next);
},
function (x, next) {
function (next) {
fs.writeFile(path.join(buildLanguagesPath, 'metadata.json'), JSON.stringify({
languages: ref.languages,
namespaces: ref.namespaces,
}), next);
},
function (next) {
next(null, ref);
},
], next);
], function (err) {
next(err, ref);
});
},
], callback);
}
function writeLanguageFile(language, namespace, translations, callback) {
var dev = global.env === 'development';
var filePath = path.join(buildLanguagesPath, language, namespace + '.json');
async.series([
async.apply(mkdirp, path.dirname(filePath)),
async.apply(fs.writeFile, filePath, JSON.stringify(translations, null, dev ? 2 : 0)),
], callback);
}
// for each language and namespace combination,
// run through core and all plugins to generate
// a full translation hash
function (ref, next) {
var languages = ref.languages;
function buildTranslations(ref, next) {
var namespaces = ref.namespaces;
var languages = ref.languages;
var plugins = _.values(Plugins.pluginsData).filter(function (plugin) {
return typeof plugin.languages === 'string';
});
var tree = {};
async.eachLimit(languages, 10, function (lang, next) {
async.eachLimit(namespaces, 10, function (namespace, next) {
async.each(namespaces, function (namespace, next) {
async.each(languages, function (lang, next) {
var translations = {};
async.series([
// core first
function (cb) {
fs.readFile(path.join(coreLanguagesPath, lang, namespace + '.json'), function (err, buffer) {
fs.readFile(path.join(coreLanguagesPath, lang, namespace + '.json'), 'utf8', function (err, file) {
if (err) {
if (err.code === 'ENOENT') {
return cb();
@@ -93,7 +102,7 @@ function getTranslationTree(callback) {
}
try {
Object.assign(translations, JSON.parse(buffer.toString()));
Object.assign(translations, JSON.parse(file));
cb();
} catch (err) {
cb(err);
@@ -106,7 +115,7 @@ function getTranslationTree(callback) {
// 2. old language string (en_GB)
// 3. corrected plugin defaultLang (en-US)
// 4. old plugin defaultLang (en_US)
async.eachLimit(plugins, 20, function (pluginData, done) {
async.each(plugins, function (pluginData, done) {
var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages);
var defaultLang = pluginData.defaultLang || 'en-GB';
@@ -116,7 +125,7 @@ function getTranslationTree(callback) {
lang.replace('-', '_').replace('-x-', '@'),
lang,
], function (language, next) {
fs.readFile(path.join(pluginLanguages, language, namespace + '.json'), function (err, buffer) {
fs.readFile(path.join(pluginLanguages, language, namespace + '.json'), 'utf8', function (err, file) {
if (err) {
if (err.code === 'ENOENT') {
return next(null, false);
@@ -125,7 +134,7 @@ function getTranslationTree(callback) {
}
try {
Object.assign(translations, JSON.parse(buffer.toString()));
Object.assign(translations, JSON.parse(file));
next(null, true);
} catch (err) {
next(err);
@@ -138,40 +147,15 @@ function getTranslationTree(callback) {
}
if (Object.keys(translations).length) {
tree[lang] = tree[lang] || {};
tree[lang][namespace] = translations;
writeLanguageFile(lang, namespace, translations, cb);
return;
}
cb();
});
},
], next);
}, next);
}, function (err) {
next(err, tree);
});
},
], callback);
}
// write translation hashes from the generated tree to language files
function writeLanguageFiles(tree, callback) {
// iterate over languages and namespaces
async.eachLimit(Object.keys(tree), 100, function (language, cb) {
var namespaces = tree[language];
async.eachLimit(Object.keys(namespaces), 10, function (namespace, next) {
var translations = namespaces[namespace];
var filePath = path.join(buildLanguagesPath, language, namespace + '.json');
mkdirp(path.dirname(filePath), function (err) {
if (err) {
return next(err);
}
fs.writeFile(filePath, JSON.stringify(translations), next);
});
}, cb);
}, callback);
}, next);
}
exports.build = function buildLanguages(callback) {
@@ -179,7 +163,7 @@ exports.build = function buildLanguages(callback) {
function (next) {
rimraf(buildLanguagesPath, next);
},
getTranslationTree,
writeLanguageFiles,
getTranslationMetadata,
buildTranslations,
], callback);
};

15
src/meta/minifier.js
View File

@@ -11,6 +11,7 @@ var autoprefixer = require('autoprefixer');
var clean = require('postcss-clean');
var fork = require('./debugFork');
require('../file'); // for graceful-fs
var Minifier = module.exports;
@@ -139,12 +140,12 @@ function executeAction(action, fork, callback) {
function concat(data, callback) {
if (data.files && data.files.length) {
async.mapLimit(data.files, 1000, function (ref, next) {
fs.readFile(ref.srcPath, function (err, buffer) {
fs.readFile(ref.srcPath, 'utf8', function (err, file) {
if (err) {
return next(err);
}
next(null, buffer.toString());
next(null, file);
});
}, function (err, files) {
if (err) {
@@ -163,18 +164,18 @@ function concat(data, callback) {
actions.concat = concat;
function minifyJS_batch(data, callback) {
async.eachLimit(data.files, 1000, function (ref, next) {
async.each(data.files, function (ref, next) {
var srcPath = ref.srcPath;
var destPath = ref.destPath;
var filename = ref.filename;
fs.readFile(srcPath, function (err, buffer) {
fs.readFile(srcPath, 'utf8', function (err, file) {
if (err) {
return next(err);
}
var scripts = {};
scripts[filename] = buffer.toString();
scripts[filename] = file;
try {
var minified = uglifyjs.minify(scripts, {
@@ -203,7 +204,7 @@ function minifyJS(data, callback) {
var srcPath = ref.srcPath;
var filename = ref.filename;
fs.readFile(srcPath, function (err, buffer) {
fs.readFile(srcPath, 'utf8', function (err, file) {
if (err) {
return next(err);
}
@@ -211,7 +212,7 @@ function minifyJS(data, callback) {
next(null, {
srcPath: srcPath,
filename: filename,
source: buffer.toString(),
source: file,
});
});
}, function (err, files) {

2
src/meta/package-install.js
View File

@@ -59,7 +59,7 @@ function preserveExtraneousPlugins() {
})
// reduce to a map of package names to package versions
.reduce(function (map, pkgName) {
var pkgConfig = JSON.parse(fs.readFileSync(path.join(modulesPath, pkgName, 'package.json')));
var pkgConfig = JSON.parse(fs.readFileSync(path.join(modulesPath, pkgName, 'package.json'), 'utf8'));
map[pkgName] = pkgConfig.version;
return map;
}, {});

8
src/meta/templates.js
View File

@@ -32,12 +32,11 @@ Templates.compile = function (callback) {
var partial = '/' + matches[1];
if (paths[partial] && relativePath !== partial) {
fs.readFile(paths[partial], function (err, file) {
fs.readFile(paths[partial], 'utf8', function (err, partialSource) {
if (err) {
return callback(err);
}
var partialSource = file.toString();
source = source.replace(regex, partialSource);
processImports(paths, relativePath, source, callback);
@@ -58,10 +57,9 @@ Templates.compile = function (callback) {
async.each(Object.keys(paths), function (relativePath, next) {
async.waterfall([
function (next) {
fs.readFile(paths[relativePath], next);
fs.readFile(paths[relativePath], 'utf8', next);
},
function (file, next) {
var source = file.toString();
function (source, next) {
processImports(paths, relativePath, source, next);
},
function (source, next) {

8
src/meta/themes.js
View File

@@ -42,7 +42,7 @@ Themes.get = function (callback) {
async.map(themes, function (theme, next) {
var config = path.join(themePath, theme, 'theme.json');
fs.readFile(config, function (err, file) {
fs.readFile(config, 'utf8', function (err, file) {
if (err) {
if (err.code === 'ENOENT') {
return next(null, null);
@@ -50,7 +50,7 @@ Themes.get = function (callback) {
return next(err);
}
try {
var configObj = JSON.parse(file.toString());
var configObj = JSON.parse(file);
// Minor adjustments for API output
configObj.type = 'local';
@@ -96,9 +96,9 @@ Themes.set = function (data, callback) {
});
},
function (next) {
fs.readFile(path.join(nconf.get('themes_path'), data.id, 'theme.json'), function (err, config) {
fs.readFile(path.join(nconf.get('themes_path'), data.id, 'theme.json'), 'utf8', function (err, config) {
if (!err) {
config = JSON.parse(config.toString());
config = JSON.parse(config);
next(null, config);
} else {
next(err);

4
src/middleware/index.js
View File

@@ -218,11 +218,11 @@ middleware.templatesOnDemand = function (req, res, next) {
return next();
}
fs.readFile(filePath.replace(/\.js$/, '.tpl'), cb);
fs.readFile(filePath.replace(/\.js$/, '.tpl'), 'utf8', cb);
},
function (source, cb) {
Benchpress.precompile({
source: source.toString(),
source: source,
minify: global.env !== 'development',
}, cb);
},

4
src/plugins/data.js
View File

@@ -33,10 +33,10 @@ Data.getPluginPaths = getPluginPaths;
function loadPluginInfo(pluginPath, callback) {
async.parallel({
package: function (next) {
fs.readFile(path.join(pluginPath, 'package.json'), next);
fs.readFile(path.join(pluginPath, 'package.json'), 'utf8', next);
},
plugin: function (next) {
fs.readFile(path.join(pluginPath, 'plugin.json'), next);
fs.readFile(path.join(pluginPath, 'plugin.json'), 'utf8', next);
},
}, function (err, results) {
if (err) {