Use less memory to build translation files (#6070)

* Change languages build to use less memory

Add graceful-fs so no ned to worry about fs limits

* Specify encoding for fs.readFile

Use eachLimit since graceful-fs handles that now

package.default.json

@@ -40,6 +40,7 @@
     "express": "^4.16.2",
     "express-session": "^1.15.6",
     "express-useragent": "1.0.8",
+    "graceful-fs": "^4.1.11",
     "html-to-text": "3.3.0",
     "ipaddr.js": "^1.5.4",
     "jimp": "0.2.28",

src/admin/search.js

@@ -63,12 +63,11 @@ var fallbackCacheInProgress = {};
 var fallbackCache = {};
 
 function initFallback(namespace, callback) {
-	fs.readFile(path.resolve(nconf.get('views_dir'), namespace + '.tpl'), function (err, file) {
+	fs.readFile(path.resolve(nconf.get('views_dir'), namespace + '.tpl'), 'utf8', function (err, template) {
 		if (err) {
 			return callback(err);
 		}
 
-		var template = file.toString();
 		var title = nsToTitle(namespace);
 
 		var translations = sanitize(template);

src/controllers/admin/settings.js

@@ -45,16 +45,16 @@ function renderEmail(req, res, next) {
 
 						async.waterfall([
 							function (next) {
-								fs.readFile(email, next);
+								fs.readFile(email, 'utf8', next);
 							},
 							function (original, next) {
-								var text = meta.config['email:custom:' + path] ? meta.config['email:custom:' + path] : original.toString();
+								var text = meta.config['email:custom:' + path] ? meta.config['email:custom:' + path] : original;
 
 								next(null, {
 									path: path,
 									fullpath: email,
 									text: text,
-									original: original.toString(),
+									original: original,
 								});
 							},
 						], next);

src/controllers/admin/themes.js

@@ -23,7 +23,7 @@ themesController.get = function (req, res, next) {
 				return next(Error('invalid-data'));
 			}
 
-			fs.readFile(themeConfigPath, next);
+			fs.readFile(themeConfigPath, 'utf8', next);
 		},
 		function (themeConfig, next) {
 			try {

src/file.js

@@ -7,9 +7,12 @@ var winston = require('winston');
 var jimp = require('jimp');
 var mkdirp = require('mkdirp');
 var mime = require('mime');
+var graceful = require('graceful-fs');
 
 var utils = require('./utils');
 
+graceful.gracefulify(fs);
+
 var file = module.exports;
 
 /**

src/image.js

@@ -120,9 +120,7 @@ image.size = function (path, callback) {
 };
 
 image.convertImageToBase64 = function (path, callback) {
-	fs.readFile(path, function (err, data) {
-		callback(err, data ? data.toString('base64') : null);
-	});
+	fs.readFile(path, 'base64', callback);
 };
 
 image.mimeFromBase64 = function (imageData) {

src/install.js

@@ -371,7 +371,7 @@ function createCategories(next) {
 
 		process.stdout.write('No categories found, populating instance with default categories\n');
 
-		fs.readFile(path.join(__dirname, '../', 'install/data/categories.json'), function (err, default_categories) {
+		fs.readFile(path.join(__dirname, '../', 'install/data/categories.json'), 'utf8', function (err, default_categories) {
 			if (err) {
 				return next(err);
 			}
@@ -402,7 +402,7 @@ function createWelcomePost(next) {
 
 	async.parallel([
 		function (next) {
-			fs.readFile(path.join(__dirname, '../', 'install/data/welcome.md'), next);
+			fs.readFile(path.join(__dirname, '../', 'install/data/welcome.md'), 'utf8', next);
 		},
 		function (next) {
 			db.getObjectField('global', 'topicCount', next);
@@ -421,7 +421,7 @@ function createWelcomePost(next) {
 				uid: 1,
 				cid: 2,
 				title: 'Welcome to your NodeBB!',
-				content: content.toString(),
+				content: content,
 			}, next);
 		} else {
 			next();
@@ -473,7 +473,7 @@ function setCopyrightWidget(next) {
 	var db = require('./database');
 	async.parallel({
 		footerJSON: function (next) {
-			fs.readFile(path.join(__dirname, '../', 'install/data/footer.json'), next);
+			fs.readFile(path.join(__dirname, '../', 'install/data/footer.json'), 'utf8', next);
 		},
 		footer: function (next) {
 			db.getObjectField('widgets:global', 'footer', next);
@@ -484,7 +484,7 @@ function setCopyrightWidget(next) {
 		}
 
 		if (!results.footer && results.footerJSON) {
-			db.setObjectField('widgets:global', 'footer', results.footerJSON.toString(), next);
+			db.setObjectField('widgets:global', 'footer', results.footerJSON, next);
 		} else {
 			next();
 		}

src/languages.js

@@ -29,7 +29,7 @@ Languages.listCodes = function (callback) {
 		return callback(null, codeCache);
 	}
 
-	fs.readFile(path.join(languagesPath, 'metadata.json'), function (err, buffer) {
+	fs.readFile(path.join(languagesPath, 'metadata.json'), 'utf8', function (err, file) {
 		if (err && err.code === 'ENOENT') {
 			return callback(null, []);
 		}
@@ -39,7 +39,7 @@ Languages.listCodes = function (callback) {
 
 		var parsed;
 		try {
-			parsed = JSON.parse(buffer.toString());
+			parsed = JSON.parse(file);
 		} catch (e) {
 			return callback(e);
 		}
@@ -64,7 +64,7 @@ Languages.list = function (callback) {
 		async.map(codes, function (folder, next) {
 			var configPath = path.join(languagesPath, folder, 'language.json');
 
-			fs.readFile(configPath, function (err, buffer) {
+			fs.readFile(configPath, 'utf8', function (err, file) {
 				if (err && err.code === 'ENOENT') {
 					return next();
 				}
@@ -72,7 +72,7 @@ Languages.list = function (callback) {
 					return next(err);
 				}
 				try {
-					var lang = JSON.parse(buffer.toString());
+					var lang = JSON.parse(file);
 					next(null, lang);
 				} catch (e) {
 					next(e);

src/meta/cacheBuster.js

@@ -31,18 +31,18 @@ exports.read = function read(callback) {
 		return callback(null, cached);
 	}
 
-	fs.readFile(filePath, function (err, buffer) {
+	fs.readFile(filePath, 'utf8', function (err, buster) {
 		if (err) {
 			winston.warn('[cache-buster] could not read cache buster', err);
 			return callback(null, generate());
 		}
 
-		if (!buffer || buffer.toString().length !== 11) {
-			winston.warn('[cache-buster] cache buster string invalid: expected /[a-z0-9]{11}/, got `' + buffer + '`');
+		if (!buster || buster.length !== 11) {
+			winston.warn('[cache-buster] cache buster string invalid: expected /[a-z0-9]{11}/, got `' + buster + '`');
 			return callback(null, generate());
 		}
 
-		cached = buffer.toString();
+		cached = buster;
 		callback(null, cached);
 	});
 };

src/meta/js.js

@@ -106,7 +106,7 @@ function minifyModules(modules, fork, callback) {
 		return prev;
 	}, []);
 
-	async.eachLimit(moduleDirs, 1000, mkdirp, function (err) {
+	async.each(moduleDirs, mkdirp, function (err) {
 		if (err) {
 			return callback(err);
 		}
@@ -126,7 +126,7 @@ function minifyModules(modules, fork, callback) {
 				minifier.js.minifyBatch(filtered.minify, fork, cb);
 			},
 			function (cb) {
-				async.eachLimit(filtered.skip, 500, function (mod, next) {
+				async.each(filtered.skip, function (mod, next) {
 					linkIfLinux(mod.srcPath, mod.destPath, next);
 				}, cb);
 			},
@@ -137,7 +137,7 @@ function minifyModules(modules, fork, callback) {
 function linkModules(callback) {
 	var modules = JS.scripts.modules;
 
-	async.eachLimit(Object.keys(modules), 1000, function (relPath, next) {
+	async.each(Object.keys(modules), function (relPath, next) {
 		var srcPath = path.join(__dirname, '../../', modules[relPath]);
 		var destPath = path.join(__dirname, '../../build/public/src/modules', relPath);
 
@@ -183,7 +183,7 @@ function getModuleList(callback) {
 	modules = modules.concat(coreDirs);
 
 	var moduleFiles = [];
-	async.eachLimit(modules, 1000, function (module, next) {
+	async.each(modules, function (module, next) {
 		var srcPath = module.srcPath;
 		var destPath = module.destPath;
 
@@ -255,7 +255,7 @@ JS.linkStatics = function (callback) {
 		if (err) {
 			return callback(err);
 		}
-		async.eachLimit(Object.keys(plugins.staticDirs), 1000, function (mappedPath, next) {
+		async.each(Object.keys(plugins.staticDirs), function (mappedPath, next) {
 			var sourceDir = plugins.staticDirs[mappedPath];
 			var destDir = path.join(__dirname, '../../build/public/plugins', mappedPath);
 

src/meta/languages.js

@@ -13,7 +13,7 @@ var Plugins = require('../plugins');
 var buildLanguagesPath = path.join(__dirname, '../../build/public/language');
 var coreLanguagesPath = path.join(__dirname, '../../public/language');
 
-function getTranslationTree(callback) {
+function getTranslationMetadata(callback) {
 	async.waterfall([
 		// generate list of languages and namespaces
 		function (next) {
@@ -49,42 +49,51 @@ function getTranslationTree(callback) {
 		// save a list of languages to `${buildLanguagesPath}/metadata.json`
 		// avoids readdirs later on
 		function (ref, next) {
-			async.waterfall([
+			async.series([
 				function (next) {
 					mkdirp(buildLanguagesPath, next);
 				},
-				function (x, next) {
+				function (next) {
 					fs.writeFile(path.join(buildLanguagesPath, 'metadata.json'), JSON.stringify({
 						languages: ref.languages,
 						namespaces: ref.namespaces,
 					}), next);
 				},
-				function (next) {
-					next(null, ref);
-				},
-			], next);
+			], function (err) {
+				next(err, ref);
+			});
 		},
+	], callback);
+}
 
-		// for each language and namespace combination,
-		// run through core and all plugins to generate
-		// a full translation hash
-		function (ref, next) {
-			var languages = ref.languages;
+function writeLanguageFile(language, namespace, translations, callback) {
+	var dev = global.env === 'development';
+	var filePath = path.join(buildLanguagesPath, language, namespace + '.json');
+
+	async.series([
+		async.apply(mkdirp, path.dirname(filePath)),
+		async.apply(fs.writeFile, filePath, JSON.stringify(translations, null, dev ? 2 : 0)),
+	], callback);
+}
+
+// for each language and namespace combination,
+// run through core and all plugins to generate
+// a full translation hash
+function buildTranslations(ref, next) {
 	var namespaces = ref.namespaces;
+	var languages = ref.languages;
 	var plugins = _.values(Plugins.pluginsData).filter(function (plugin) {
 		return typeof plugin.languages === 'string';
 	});
 
-			var tree = {};
-
-			async.eachLimit(languages, 10, function (lang, next) {
-				async.eachLimit(namespaces, 10, function (namespace, next) {
+	async.each(namespaces, function (namespace, next) {
+		async.each(languages, function (lang, next) {
 			var translations = {};
 
 			async.series([
 				// core first
 				function (cb) {
-							fs.readFile(path.join(coreLanguagesPath, lang, namespace + '.json'), function (err, buffer) {
+					fs.readFile(path.join(coreLanguagesPath, lang, namespace + '.json'), 'utf8', function (err, file) {
 						if (err) {
 							if (err.code === 'ENOENT') {
 								return cb();
@@ -93,7 +102,7 @@ function getTranslationTree(callback) {
 						}
 
 						try {
-									Object.assign(translations, JSON.parse(buffer.toString()));
+							Object.assign(translations, JSON.parse(file));
 							cb();
 						} catch (err) {
 							cb(err);
@@ -106,7 +115,7 @@ function getTranslationTree(callback) {
 					//  2. old language string (en_GB)
 					//  3. corrected plugin defaultLang (en-US)
 					//  4. old plugin defaultLang (en_US)
-							async.eachLimit(plugins, 20, function (pluginData, done) {
+					async.each(plugins, function (pluginData, done) {
 						var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages);
 						var defaultLang = pluginData.defaultLang || 'en-GB';
 
@@ -116,7 +125,7 @@ function getTranslationTree(callback) {
 							lang.replace('-', '_').replace('-x-', '@'),
 							lang,
 						], function (language, next) {
-									fs.readFile(path.join(pluginLanguages, language, namespace + '.json'), function (err, buffer) {
+							fs.readFile(path.join(pluginLanguages, language, namespace + '.json'), 'utf8', function (err, file) {
 								if (err) {
 									if (err.code === 'ENOENT') {
 										return next(null, false);
@@ -125,7 +134,7 @@ function getTranslationTree(callback) {
 								}
 
 								try {
-											Object.assign(translations, JSON.parse(buffer.toString()));
+									Object.assign(translations, JSON.parse(file));
 									next(null, true);
 								} catch (err) {
 									next(err);
@@ -138,40 +147,15 @@ function getTranslationTree(callback) {
 						}
 
 						if (Object.keys(translations).length) {
-									tree[lang] = tree[lang] || {};
-									tree[lang][namespace] = translations;
+							writeLanguageFile(lang, namespace, translations, cb);
+							return;
 						}
 						cb();
 					});
 				},
 			], next);
 		}, next);
-			}, function (err) {
-				next(err, tree);
-			});
-		},
-	], callback);
-}
-
-// write translation hashes from the generated tree to language files
-function writeLanguageFiles(tree, callback) {
-	// iterate over languages and namespaces
-	async.eachLimit(Object.keys(tree), 100, function (language, cb) {
-		var namespaces = tree[language];
-		async.eachLimit(Object.keys(namespaces), 10, function (namespace, next) {
-			var translations = namespaces[namespace];
-
-			var filePath = path.join(buildLanguagesPath, language, namespace + '.json');
-
-			mkdirp(path.dirname(filePath), function (err) {
-				if (err) {
-					return next(err);
-				}
-
-				fs.writeFile(filePath, JSON.stringify(translations), next);
-			});
-		}, cb);
-	}, callback);
+	}, next);
 }
 
 exports.build = function buildLanguages(callback) {
@@ -179,7 +163,7 @@ exports.build = function buildLanguages(callback) {
 		function (next) {
 			rimraf(buildLanguagesPath, next);
 		},
-		getTranslationTree,
-		writeLanguageFiles,
+		getTranslationMetadata,
+		buildTranslations,
 	], callback);
 };

src/meta/minifier.js

@@ -11,6 +11,7 @@ var autoprefixer = require('autoprefixer');
 var clean = require('postcss-clean');
 
 var fork = require('./debugFork');
+require('../file'); // for graceful-fs
 
 var Minifier = module.exports;
 
@@ -139,12 +140,12 @@ function executeAction(action, fork, callback) {
 function concat(data, callback) {
 	if (data.files && data.files.length) {
 		async.mapLimit(data.files, 1000, function (ref, next) {
-			fs.readFile(ref.srcPath, function (err, buffer) {
+			fs.readFile(ref.srcPath, 'utf8', function (err, file) {
 				if (err) {
 					return next(err);
 				}
 
-				next(null, buffer.toString());
+				next(null, file);
 			});
 		}, function (err, files) {
 			if (err) {
@@ -163,18 +164,18 @@ function concat(data, callback) {
 actions.concat = concat;
 
 function minifyJS_batch(data, callback) {
-	async.eachLimit(data.files, 1000, function (ref, next) {
+	async.each(data.files, function (ref, next) {
 		var srcPath = ref.srcPath;
 		var destPath = ref.destPath;
 		var filename = ref.filename;
 
-		fs.readFile(srcPath, function (err, buffer) {
+		fs.readFile(srcPath, 'utf8', function (err, file) {
 			if (err) {
 				return next(err);
 			}
 
 			var scripts = {};
-			scripts[filename] = buffer.toString();
+			scripts[filename] = file;
 
 			try {
 				var minified = uglifyjs.minify(scripts, {
@@ -203,7 +204,7 @@ function minifyJS(data, callback) {
 		var srcPath = ref.srcPath;
 		var filename = ref.filename;
 
-		fs.readFile(srcPath, function (err, buffer) {
+		fs.readFile(srcPath, 'utf8', function (err, file) {
 			if (err) {
 				return next(err);
 			}
@@ -211,7 +212,7 @@ function minifyJS(data, callback) {
 			next(null, {
 				srcPath: srcPath,
 				filename: filename,
-				source: buffer.toString(),
+				source: file,
 			});
 		});
 	}, function (err, files) {

src/meta/package-install.js

@@ -59,7 +59,7 @@ function preserveExtraneousPlugins() {
 		})
 		// reduce to a map of package names to package versions
 		.reduce(function (map, pkgName) {
-			var pkgConfig = JSON.parse(fs.readFileSync(path.join(modulesPath, pkgName, 'package.json')));
+			var pkgConfig = JSON.parse(fs.readFileSync(path.join(modulesPath, pkgName, 'package.json'), 'utf8'));
 			map[pkgName] = pkgConfig.version;
 			return map;
 		}, {});

src/meta/templates.js

@@ -32,12 +32,11 @@ Templates.compile = function (callback) {
 
 		var partial = '/' + matches[1];
 		if (paths[partial] && relativePath !== partial) {
-			fs.readFile(paths[partial], function (err, file) {
+			fs.readFile(paths[partial], 'utf8', function (err, partialSource) {
 				if (err) {
 					return callback(err);
 				}
 
-				var partialSource = file.toString();
 				source = source.replace(regex, partialSource);
 
 				processImports(paths, relativePath, source, callback);
@@ -58,10 +57,9 @@ Templates.compile = function (callback) {
 			async.each(Object.keys(paths), function (relativePath, next) {
 				async.waterfall([
 					function (next) {
-						fs.readFile(paths[relativePath], next);
+						fs.readFile(paths[relativePath], 'utf8', next);
 					},
-					function (file, next) {
-						var source = file.toString();
+					function (source, next) {
 						processImports(paths, relativePath, source, next);
 					},
 					function (source, next) {

src/meta/themes.js

@@ -42,7 +42,7 @@ Themes.get = function (callback) {
 			async.map(themes, function (theme, next) {
 				var config = path.join(themePath, theme, 'theme.json');
 
-				fs.readFile(config, function (err, file) {
+				fs.readFile(config, 'utf8', function (err, file) {
 					if (err) {
 						if (err.code === 'ENOENT') {
 							return next(null, null);
@@ -50,7 +50,7 @@ Themes.get = function (callback) {
 						return next(err);
 					}
 					try {
-						var configObj = JSON.parse(file.toString());
+						var configObj = JSON.parse(file);
 
 						// Minor adjustments for API output
 						configObj.type = 'local';
@@ -96,9 +96,9 @@ Themes.set = function (data, callback) {
 				});
 			},
 			function (next) {
-				fs.readFile(path.join(nconf.get('themes_path'), data.id, 'theme.json'), function (err, config) {
+				fs.readFile(path.join(nconf.get('themes_path'), data.id, 'theme.json'), 'utf8', function (err, config) {
 					if (!err) {
-						config = JSON.parse(config.toString());
+						config = JSON.parse(config);
 						next(null, config);
 					} else {
 						next(err);

src/middleware/index.js

@@ -218,11 +218,11 @@ middleware.templatesOnDemand = function (req, res, next) {
 				return next();
 			}
 
-			fs.readFile(filePath.replace(/\.js$/, '.tpl'), cb);
+			fs.readFile(filePath.replace(/\.js$/, '.tpl'), 'utf8', cb);
 		},
 		function (source, cb) {
 			Benchpress.precompile({
-				source: source.toString(),
+				source: source,
 				minify: global.env !== 'development',
 			}, cb);
 		},

src/plugins/data.js

@@ -33,10 +33,10 @@ Data.getPluginPaths = getPluginPaths;
 function loadPluginInfo(pluginPath, callback) {
 	async.parallel({
 		package: function (next) {
-			fs.readFile(path.join(pluginPath, 'package.json'), next);
+			fs.readFile(path.join(pluginPath, 'package.json'), 'utf8', next);
 		},
 		plugin: function (next) {
-			fs.readFile(path.join(pluginPath, 'plugin.json'), next);
+			fs.readFile(path.join(pluginPath, 'plugin.json'), 'utf8', next);
 		},
 	}, function (err, results) {
 		if (err) {