From c47c47f7e3d9c3595d9383c1346be6e6b64134da Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 16 Nov 2017 15:43:52 -0700 Subject: [PATCH] Use less memory to build translation files (#6070) * Change languages build to use less memory Add graceful-fs so no ned to worry about fs limits * Specify encoding for fs.readFile Use eachLimit since graceful-fs handles that now --- package.default.json | 1 + src/admin/search.js | 3 +- src/controllers/admin/settings.js | 6 +- src/controllers/admin/themes.js | 2 +- src/file.js | 3 + src/image.js | 4 +- src/install.js | 10 +- src/languages.js | 8 +- src/meta/cacheBuster.js | 8 +- src/meta/js.js | 10 +- src/meta/languages.js | 194 ++++++++++++++---------------- src/meta/minifier.js | 15 +-- src/meta/package-install.js | 2 +- src/meta/templates.js | 8 +- src/meta/themes.js | 8 +- src/middleware/index.js | 4 +- src/plugins/data.js | 4 +- 17 files changed, 137 insertions(+), 153 deletions(-) diff --git a/package.default.json b/package.default.json index 10b8973b82..ce6780128f 100644 --- a/package.default.json +++ b/package.default.json @@ -40,6 +40,7 @@ "express": "^4.16.2", "express-session": "^1.15.6", "express-useragent": "1.0.8", + "graceful-fs": "^4.1.11", "html-to-text": "3.3.0", "ipaddr.js": "^1.5.4", "jimp": "0.2.28", diff --git a/src/admin/search.js b/src/admin/search.js index 1803f3298c..6f9792138e 100644 --- a/src/admin/search.js +++ b/src/admin/search.js @@ -63,12 +63,11 @@ var fallbackCacheInProgress = {}; var fallbackCache = {}; function initFallback(namespace, callback) { - fs.readFile(path.resolve(nconf.get('views_dir'), namespace + '.tpl'), function (err, file) { + fs.readFile(path.resolve(nconf.get('views_dir'), namespace + '.tpl'), 'utf8', function (err, template) { if (err) { return callback(err); } - var template = file.toString(); var title = nsToTitle(namespace); var translations = sanitize(template); diff --git a/src/controllers/admin/settings.js b/src/controllers/admin/settings.js index 28322892f3..8944655881 100644 --- a/src/controllers/admin/settings.js +++ b/src/controllers/admin/settings.js @@ -45,16 +45,16 @@ function renderEmail(req, res, next) { async.waterfall([ function (next) { - fs.readFile(email, next); + fs.readFile(email, 'utf8', next); }, function (original, next) { - var text = meta.config['email:custom:' + path] ? meta.config['email:custom:' + path] : original.toString(); + var text = meta.config['email:custom:' + path] ? meta.config['email:custom:' + path] : original; next(null, { path: path, fullpath: email, text: text, - original: original.toString(), + original: original, }); }, ], next); diff --git a/src/controllers/admin/themes.js b/src/controllers/admin/themes.js index 850feb201d..717d4bf0dc 100644 --- a/src/controllers/admin/themes.js +++ b/src/controllers/admin/themes.js @@ -23,7 +23,7 @@ themesController.get = function (req, res, next) { return next(Error('invalid-data')); } - fs.readFile(themeConfigPath, next); + fs.readFile(themeConfigPath, 'utf8', next); }, function (themeConfig, next) { try { diff --git a/src/file.js b/src/file.js index c091da614f..8027f7ad77 100644 --- a/src/file.js +++ b/src/file.js @@ -7,9 +7,12 @@ var winston = require('winston'); var jimp = require('jimp'); var mkdirp = require('mkdirp'); var mime = require('mime'); +var graceful = require('graceful-fs'); var utils = require('./utils'); +graceful.gracefulify(fs); + var file = module.exports; /** diff --git a/src/image.js b/src/image.js index fc3ddd5e76..f2afb21003 100644 --- a/src/image.js +++ b/src/image.js @@ -120,9 +120,7 @@ image.size = function (path, callback) { }; image.convertImageToBase64 = function (path, callback) { - fs.readFile(path, function (err, data) { - callback(err, data ? data.toString('base64') : null); - }); + fs.readFile(path, 'base64', callback); }; image.mimeFromBase64 = function (imageData) { diff --git a/src/install.js b/src/install.js index 3a69595599..fecf86b379 100644 --- a/src/install.js +++ b/src/install.js @@ -371,7 +371,7 @@ function createCategories(next) { process.stdout.write('No categories found, populating instance with default categories\n'); - fs.readFile(path.join(__dirname, '../', 'install/data/categories.json'), function (err, default_categories) { + fs.readFile(path.join(__dirname, '../', 'install/data/categories.json'), 'utf8', function (err, default_categories) { if (err) { return next(err); } @@ -402,7 +402,7 @@ function createWelcomePost(next) { async.parallel([ function (next) { - fs.readFile(path.join(__dirname, '../', 'install/data/welcome.md'), next); + fs.readFile(path.join(__dirname, '../', 'install/data/welcome.md'), 'utf8', next); }, function (next) { db.getObjectField('global', 'topicCount', next); @@ -421,7 +421,7 @@ function createWelcomePost(next) { uid: 1, cid: 2, title: 'Welcome to your NodeBB!', - content: content.toString(), + content: content, }, next); } else { next(); @@ -473,7 +473,7 @@ function setCopyrightWidget(next) { var db = require('./database'); async.parallel({ footerJSON: function (next) { - fs.readFile(path.join(__dirname, '../', 'install/data/footer.json'), next); + fs.readFile(path.join(__dirname, '../', 'install/data/footer.json'), 'utf8', next); }, footer: function (next) { db.getObjectField('widgets:global', 'footer', next); @@ -484,7 +484,7 @@ function setCopyrightWidget(next) { } if (!results.footer && results.footerJSON) { - db.setObjectField('widgets:global', 'footer', results.footerJSON.toString(), next); + db.setObjectField('widgets:global', 'footer', results.footerJSON, next); } else { next(); } diff --git a/src/languages.js b/src/languages.js index 77945bacc6..cdf56bf81d 100644 --- a/src/languages.js +++ b/src/languages.js @@ -29,7 +29,7 @@ Languages.listCodes = function (callback) { return callback(null, codeCache); } - fs.readFile(path.join(languagesPath, 'metadata.json'), function (err, buffer) { + fs.readFile(path.join(languagesPath, 'metadata.json'), 'utf8', function (err, file) { if (err && err.code === 'ENOENT') { return callback(null, []); } @@ -39,7 +39,7 @@ Languages.listCodes = function (callback) { var parsed; try { - parsed = JSON.parse(buffer.toString()); + parsed = JSON.parse(file); } catch (e) { return callback(e); } @@ -64,7 +64,7 @@ Languages.list = function (callback) { async.map(codes, function (folder, next) { var configPath = path.join(languagesPath, folder, 'language.json'); - fs.readFile(configPath, function (err, buffer) { + fs.readFile(configPath, 'utf8', function (err, file) { if (err && err.code === 'ENOENT') { return next(); } @@ -72,7 +72,7 @@ Languages.list = function (callback) { return next(err); } try { - var lang = JSON.parse(buffer.toString()); + var lang = JSON.parse(file); next(null, lang); } catch (e) { next(e); diff --git a/src/meta/cacheBuster.js b/src/meta/cacheBuster.js index f88cebb680..4ea3109dbe 100644 --- a/src/meta/cacheBuster.js +++ b/src/meta/cacheBuster.js @@ -31,18 +31,18 @@ exports.read = function read(callback) { return callback(null, cached); } - fs.readFile(filePath, function (err, buffer) { + fs.readFile(filePath, 'utf8', function (err, buster) { if (err) { winston.warn('[cache-buster] could not read cache buster', err); return callback(null, generate()); } - if (!buffer || buffer.toString().length !== 11) { - winston.warn('[cache-buster] cache buster string invalid: expected /[a-z0-9]{11}/, got `' + buffer + '`'); + if (!buster || buster.length !== 11) { + winston.warn('[cache-buster] cache buster string invalid: expected /[a-z0-9]{11}/, got `' + buster + '`'); return callback(null, generate()); } - cached = buffer.toString(); + cached = buster; callback(null, cached); }); }; diff --git a/src/meta/js.js b/src/meta/js.js index 28b114434a..cac47834be 100644 --- a/src/meta/js.js +++ b/src/meta/js.js @@ -106,7 +106,7 @@ function minifyModules(modules, fork, callback) { return prev; }, []); - async.eachLimit(moduleDirs, 1000, mkdirp, function (err) { + async.each(moduleDirs, mkdirp, function (err) { if (err) { return callback(err); } @@ -126,7 +126,7 @@ function minifyModules(modules, fork, callback) { minifier.js.minifyBatch(filtered.minify, fork, cb); }, function (cb) { - async.eachLimit(filtered.skip, 500, function (mod, next) { + async.each(filtered.skip, function (mod, next) { linkIfLinux(mod.srcPath, mod.destPath, next); }, cb); }, @@ -137,7 +137,7 @@ function minifyModules(modules, fork, callback) { function linkModules(callback) { var modules = JS.scripts.modules; - async.eachLimit(Object.keys(modules), 1000, function (relPath, next) { + async.each(Object.keys(modules), function (relPath, next) { var srcPath = path.join(__dirname, '../../', modules[relPath]); var destPath = path.join(__dirname, '../../build/public/src/modules', relPath); @@ -183,7 +183,7 @@ function getModuleList(callback) { modules = modules.concat(coreDirs); var moduleFiles = []; - async.eachLimit(modules, 1000, function (module, next) { + async.each(modules, function (module, next) { var srcPath = module.srcPath; var destPath = module.destPath; @@ -255,7 +255,7 @@ JS.linkStatics = function (callback) { if (err) { return callback(err); } - async.eachLimit(Object.keys(plugins.staticDirs), 1000, function (mappedPath, next) { + async.each(Object.keys(plugins.staticDirs), function (mappedPath, next) { var sourceDir = plugins.staticDirs[mappedPath]; var destDir = path.join(__dirname, '../../build/public/plugins', mappedPath); diff --git a/src/meta/languages.js b/src/meta/languages.js index 3b9f3c3a9e..fc907e60be 100644 --- a/src/meta/languages.js +++ b/src/meta/languages.js @@ -13,7 +13,7 @@ var Plugins = require('../plugins'); var buildLanguagesPath = path.join(__dirname, '../../build/public/language'); var coreLanguagesPath = path.join(__dirname, '../../public/language'); -function getTranslationTree(callback) { +function getTranslationMetadata(callback) { async.waterfall([ // generate list of languages and namespaces function (next) { @@ -49,129 +49,113 @@ function getTranslationTree(callback) { // save a list of languages to `${buildLanguagesPath}/metadata.json` // avoids readdirs later on function (ref, next) { - async.waterfall([ + async.series([ function (next) { mkdirp(buildLanguagesPath, next); }, - function (x, next) { + function (next) { fs.writeFile(path.join(buildLanguagesPath, 'metadata.json'), JSON.stringify({ languages: ref.languages, namespaces: ref.namespaces, }), next); }, - function (next) { - next(null, ref); - }, - ], next); - }, - - // for each language and namespace combination, - // run through core and all plugins to generate - // a full translation hash - function (ref, next) { - var languages = ref.languages; - var namespaces = ref.namespaces; - var plugins = _.values(Plugins.pluginsData).filter(function (plugin) { - return typeof plugin.languages === 'string'; + ], function (err) { + next(err, ref); }); + }, + ], callback); +} - var tree = {}; +function writeLanguageFile(language, namespace, translations, callback) { + var dev = global.env === 'development'; + var filePath = path.join(buildLanguagesPath, language, namespace + '.json'); - async.eachLimit(languages, 10, function (lang, next) { - async.eachLimit(namespaces, 10, function (namespace, next) { - var translations = {}; + async.series([ + async.apply(mkdirp, path.dirname(filePath)), + async.apply(fs.writeFile, filePath, JSON.stringify(translations, null, dev ? 2 : 0)), + ], callback); +} - async.series([ - // core first - function (cb) { - fs.readFile(path.join(coreLanguagesPath, lang, namespace + '.json'), function (err, buffer) { +// for each language and namespace combination, +// run through core and all plugins to generate +// a full translation hash +function buildTranslations(ref, next) { + var namespaces = ref.namespaces; + var languages = ref.languages; + var plugins = _.values(Plugins.pluginsData).filter(function (plugin) { + return typeof plugin.languages === 'string'; + }); + + async.each(namespaces, function (namespace, next) { + async.each(languages, function (lang, next) { + var translations = {}; + + async.series([ + // core first + function (cb) { + fs.readFile(path.join(coreLanguagesPath, lang, namespace + '.json'), 'utf8', function (err, file) { + if (err) { + if (err.code === 'ENOENT') { + return cb(); + } + return cb(err); + } + + try { + Object.assign(translations, JSON.parse(file)); + cb(); + } catch (err) { + cb(err); + } + }); + }, + function (cb) { + // for each plugin, fallback in this order: + // 1. correct language string (en-GB) + // 2. old language string (en_GB) + // 3. corrected plugin defaultLang (en-US) + // 4. old plugin defaultLang (en_US) + async.each(plugins, function (pluginData, done) { + var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); + var defaultLang = pluginData.defaultLang || 'en-GB'; + + async.eachSeries([ + defaultLang.replace('-', '_').replace('-x-', '@'), + defaultLang.replace('_', '-').replace('@', '-x-'), + lang.replace('-', '_').replace('-x-', '@'), + lang, + ], function (language, next) { + fs.readFile(path.join(pluginLanguages, language, namespace + '.json'), 'utf8', function (err, file) { if (err) { if (err.code === 'ENOENT') { - return cb(); + return next(null, false); } - return cb(err); + return next(err); } try { - Object.assign(translations, JSON.parse(buffer.toString())); - cb(); + Object.assign(translations, JSON.parse(file)); + next(null, true); } catch (err) { - cb(err); - } - }); - }, - function (cb) { - // for each plugin, fallback in this order: - // 1. correct language string (en-GB) - // 2. old language string (en_GB) - // 3. corrected plugin defaultLang (en-US) - // 4. old plugin defaultLang (en_US) - async.eachLimit(plugins, 20, function (pluginData, done) { - var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); - var defaultLang = pluginData.defaultLang || 'en-GB'; - - async.eachSeries([ - defaultLang.replace('-', '_').replace('-x-', '@'), - defaultLang.replace('_', '-').replace('@', '-x-'), - lang.replace('-', '_').replace('-x-', '@'), - lang, - ], function (language, next) { - fs.readFile(path.join(pluginLanguages, language, namespace + '.json'), function (err, buffer) { - if (err) { - if (err.code === 'ENOENT') { - return next(null, false); - } - return next(err); - } - - try { - Object.assign(translations, JSON.parse(buffer.toString())); - next(null, true); - } catch (err) { - next(err); - } - }); - }, done); - }, function (err) { - if (err) { - return cb(err); - } - - if (Object.keys(translations).length) { - tree[lang] = tree[lang] || {}; - tree[lang][namespace] = translations; + next(err); } - cb(); }); - }, - ], next); - }, next); - }, function (err) { - next(err, tree); - }); - }, - ], callback); -} - -// write translation hashes from the generated tree to language files -function writeLanguageFiles(tree, callback) { - // iterate over languages and namespaces - async.eachLimit(Object.keys(tree), 100, function (language, cb) { - var namespaces = tree[language]; - async.eachLimit(Object.keys(namespaces), 10, function (namespace, next) { - var translations = namespaces[namespace]; - - var filePath = path.join(buildLanguagesPath, language, namespace + '.json'); - - mkdirp(path.dirname(filePath), function (err) { - if (err) { - return next(err); - } - - fs.writeFile(filePath, JSON.stringify(translations), next); - }); - }, cb); - }, callback); + }, done); + }, function (err) { + if (err) { + return cb(err); + } + + if (Object.keys(translations).length) { + writeLanguageFile(lang, namespace, translations, cb); + return; + } + cb(); + }); + }, + ], next); + }, next); + }, next); } exports.build = function buildLanguages(callback) { @@ -179,7 +163,7 @@ exports.build = function buildLanguages(callback) { function (next) { rimraf(buildLanguagesPath, next); }, - getTranslationTree, - writeLanguageFiles, + getTranslationMetadata, + buildTranslations, ], callback); }; diff --git a/src/meta/minifier.js b/src/meta/minifier.js index c51f41578b..ed6a68a625 100644 --- a/src/meta/minifier.js +++ b/src/meta/minifier.js @@ -11,6 +11,7 @@ var autoprefixer = require('autoprefixer'); var clean = require('postcss-clean'); var fork = require('./debugFork'); +require('../file'); // for graceful-fs var Minifier = module.exports; @@ -139,12 +140,12 @@ function executeAction(action, fork, callback) { function concat(data, callback) { if (data.files && data.files.length) { async.mapLimit(data.files, 1000, function (ref, next) { - fs.readFile(ref.srcPath, function (err, buffer) { + fs.readFile(ref.srcPath, 'utf8', function (err, file) { if (err) { return next(err); } - next(null, buffer.toString()); + next(null, file); }); }, function (err, files) { if (err) { @@ -163,18 +164,18 @@ function concat(data, callback) { actions.concat = concat; function minifyJS_batch(data, callback) { - async.eachLimit(data.files, 1000, function (ref, next) { + async.each(data.files, function (ref, next) { var srcPath = ref.srcPath; var destPath = ref.destPath; var filename = ref.filename; - fs.readFile(srcPath, function (err, buffer) { + fs.readFile(srcPath, 'utf8', function (err, file) { if (err) { return next(err); } var scripts = {}; - scripts[filename] = buffer.toString(); + scripts[filename] = file; try { var minified = uglifyjs.minify(scripts, { @@ -203,7 +204,7 @@ function minifyJS(data, callback) { var srcPath = ref.srcPath; var filename = ref.filename; - fs.readFile(srcPath, function (err, buffer) { + fs.readFile(srcPath, 'utf8', function (err, file) { if (err) { return next(err); } @@ -211,7 +212,7 @@ function minifyJS(data, callback) { next(null, { srcPath: srcPath, filename: filename, - source: buffer.toString(), + source: file, }); }); }, function (err, files) { diff --git a/src/meta/package-install.js b/src/meta/package-install.js index 3fee4cb9e4..ca465493bd 100644 --- a/src/meta/package-install.js +++ b/src/meta/package-install.js @@ -59,7 +59,7 @@ function preserveExtraneousPlugins() { }) // reduce to a map of package names to package versions .reduce(function (map, pkgName) { - var pkgConfig = JSON.parse(fs.readFileSync(path.join(modulesPath, pkgName, 'package.json'))); + var pkgConfig = JSON.parse(fs.readFileSync(path.join(modulesPath, pkgName, 'package.json'), 'utf8')); map[pkgName] = pkgConfig.version; return map; }, {}); diff --git a/src/meta/templates.js b/src/meta/templates.js index 4f9f59d861..03097ef9dd 100644 --- a/src/meta/templates.js +++ b/src/meta/templates.js @@ -32,12 +32,11 @@ Templates.compile = function (callback) { var partial = '/' + matches[1]; if (paths[partial] && relativePath !== partial) { - fs.readFile(paths[partial], function (err, file) { + fs.readFile(paths[partial], 'utf8', function (err, partialSource) { if (err) { return callback(err); } - var partialSource = file.toString(); source = source.replace(regex, partialSource); processImports(paths, relativePath, source, callback); @@ -58,10 +57,9 @@ Templates.compile = function (callback) { async.each(Object.keys(paths), function (relativePath, next) { async.waterfall([ function (next) { - fs.readFile(paths[relativePath], next); + fs.readFile(paths[relativePath], 'utf8', next); }, - function (file, next) { - var source = file.toString(); + function (source, next) { processImports(paths, relativePath, source, next); }, function (source, next) { diff --git a/src/meta/themes.js b/src/meta/themes.js index 2ebc091036..e6eeb7011d 100644 --- a/src/meta/themes.js +++ b/src/meta/themes.js @@ -42,7 +42,7 @@ Themes.get = function (callback) { async.map(themes, function (theme, next) { var config = path.join(themePath, theme, 'theme.json'); - fs.readFile(config, function (err, file) { + fs.readFile(config, 'utf8', function (err, file) { if (err) { if (err.code === 'ENOENT') { return next(null, null); @@ -50,7 +50,7 @@ Themes.get = function (callback) { return next(err); } try { - var configObj = JSON.parse(file.toString()); + var configObj = JSON.parse(file); // Minor adjustments for API output configObj.type = 'local'; @@ -96,9 +96,9 @@ Themes.set = function (data, callback) { }); }, function (next) { - fs.readFile(path.join(nconf.get('themes_path'), data.id, 'theme.json'), function (err, config) { + fs.readFile(path.join(nconf.get('themes_path'), data.id, 'theme.json'), 'utf8', function (err, config) { if (!err) { - config = JSON.parse(config.toString()); + config = JSON.parse(config); next(null, config); } else { next(err); diff --git a/src/middleware/index.js b/src/middleware/index.js index d5f7e5e6a3..28ce7c3e10 100644 --- a/src/middleware/index.js +++ b/src/middleware/index.js @@ -218,11 +218,11 @@ middleware.templatesOnDemand = function (req, res, next) { return next(); } - fs.readFile(filePath.replace(/\.js$/, '.tpl'), cb); + fs.readFile(filePath.replace(/\.js$/, '.tpl'), 'utf8', cb); }, function (source, cb) { Benchpress.precompile({ - source: source.toString(), + source: source, minify: global.env !== 'development', }, cb); }, diff --git a/src/plugins/data.js b/src/plugins/data.js index b793a365d0..0dd7fa3a67 100644 --- a/src/plugins/data.js +++ b/src/plugins/data.js @@ -33,10 +33,10 @@ Data.getPluginPaths = getPluginPaths; function loadPluginInfo(pluginPath, callback) { async.parallel({ package: function (next) { - fs.readFile(path.join(pluginPath, 'package.json'), next); + fs.readFile(path.join(pluginPath, 'package.json'), 'utf8', next); }, plugin: function (next) { - fs.readFile(path.join(pluginPath, 'plugin.json'), next); + fs.readFile(path.join(pluginPath, 'plugin.json'), 'utf8', next); }, }, function (err, results) { if (err) {