Merge pull request #5690 from NodeBB/build-refactor

Restrict total threads, minify modules in a batch, link instead of copying pre-minified files

src/meta/build.js

@@ -2,7 +2,6 @@
 
 var async = require('async');
 var winston = require('winston');
-var os = require('os');
 var nconf = require('nconf');
 var padstart = require('lodash.padstart');
 
@@ -181,7 +180,7 @@ function build(targets, callback) {
 	async.series([
 		beforeBuild,
 		function (next) {
-			var parallel = os.cpus().length > 1 && !nconf.get('series');
+			var parallel = !nconf.get('series');
 			if (parallel) {
 				winston.info('[build] Building in parallel mode');
 			} else {

src/meta/js.js

@@ -89,43 +89,40 @@ module.exports = function (Meta) {
 	};
 
 	function minifyModules(modules, fork, callback) {
-		// for it to never fork
+		var moduleDirs = modules.reduce(function (prev, mod) {
-		// otherwise it spawns way too many processes
+			var dir = path.resolve(path.dirname(mod.destPath));
-		// maybe eventually we can pool modules
+			if (prev.indexOf(dir) === -1) {
-		// and pass the pools to the minifer
+				prev.push(dir);
-		// to reduce the total number of threads
+			}
-		fork = false;
+			return prev;
+		}, []);
 
-		async.eachLimit(modules, 500, function (mod, next) {
+		async.eachLimit(moduleDirs, 1000, mkdirp, function (err) {
-			var srcPath = mod.srcPath;
+			if (err) {
-			var destPath = mod.destPath;
+				return callback(err);
+			}
 
-			async.parallel({
+			var filtered = modules.reduce(function (prev, mod) {
-				dirped: function (cb) {
+				if (mod.srcPath.endsWith('.min.js') || path.dirname(mod.srcPath).endsWith('min')) {
-					mkdirp(path.dirname(destPath), cb);
+					prev.skip.push(mod);
-				},
+				} else {
-				minified: function (cb) {
+					prev.minify.push(mod);
-					fs.readFile(srcPath, function (err, buffer) {
+				}
-						if (err) {
-							return cb(err);
-						}
 
-						if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) {
+				return prev;
-							return cb(null, { code: buffer.toString() });
+			}, { minify: [], skip: [] });
-						}
 
-						minifier.js.minify(buffer.toString(), fork, cb);
+			async.parallel([
-					});
+				function (cb) {
+					minifier.js.minifyBatch(filtered.minify, fork, cb);
 				},
-			}, function (err, results) {
+				function (cb) {
-				if (err) {
+					async.eachLimit(filtered.skip, 500, function (mod, next) {
-					return next(err);
+						file.link(mod.srcPath, mod.destPath, next);
-				}
+					}, cb);
-
+				},
-				var minified = results.minified;
+			], callback);
-				fs.writeFile(destPath, minified.code, next);
+		});
-			});
-		}, callback);
 	}
 
 	function linkModules(callback) {

src/meta/minifier.js

@@ -5,6 +5,7 @@ var async = require('async');
 var fs = require('fs');
 var childProcess = require('child_process');
 var os = require('os');
+var winston = require('winston');
 var less = require('less');
 var postcss = require('postcss');
 var autoprefixer = require('autoprefixer');
@@ -37,23 +38,26 @@ function setupDebugging() {
 	return forkProcessParams;
 }
 
-var children = [];
+var pool = [];
+var free = [];
+
+Minifier.maxThreads = os.cpus().length - 1;
+
+winston.verbose('[minifier] utilizing a maximum of ' + Minifier.maxThreads + ' additional threads');
 
 Minifier.killAll = function () {
-	children.forEach(function (child) {
+	pool.forEach(function (child) {
 		child.kill('SIGTERM');
 	});
 
-	children = [];
+	pool.length = 0;
 };
 
-function removeChild(proc) {
+function getChild() {
-	children = children.filter(function (child) {
+	if (free.length) {
-		return child !== proc;
+		return free.shift();
-	});
+	}
-}
 
-function forkAction(action, callback) {
 	var forkProcessParams = setupDebugging();
 	var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, {
 		cwd: __dirname,
@@ -61,17 +65,32 @@ function forkAction(action, callback) {
 			minifier_child: true,
 		},
 	}));
+	pool.push(proc);
 
-	children.push(proc);
+	return proc;
+}
+
+function freeChild(proc) {
+	proc.removeAllListeners();
+	free.push(proc);
+}
+
+function removeChild(proc) {
+	var i = pool.indexOf(proc);
+	pool.splice(i, 1);
+}
+
+function forkAction(action, callback) {
+	var proc = getChild();
 
 	proc.on('message', function (message) {
+		freeChild(proc);
+
 		if (message.type === 'error') {
-			proc.kill();
+			return callback(message.err);
-			return callback(new Error(message.message));
 		}
 
 		if (message.type === 'end') {
-			proc.kill();
 			callback(null, message.result);
 		}
 	});
@@ -85,10 +104,6 @@ function forkAction(action, callback) {
 		type: 'action',
 		action: action,
 	});
-
-	proc.on('close', function () {
-		removeChild(proc);
-	});
 }
 
 var actions = {};
@@ -100,7 +115,7 @@ if (process.env.minifier_child) {
 			if (typeof actions[action.act] !== 'function') {
 				process.send({
 					type: 'error',
-					message: 'Unknown action',
+					err: Error('Unknown action'),
 				});
 				return;
 			}
@@ -109,7 +124,7 @@ if (process.env.minifier_child) {
 				if (err) {
 					process.send({
 						type: 'error',
-						message: err.message,
+						err: err,
 					});
 					return;
 				}
@@ -124,7 +139,7 @@ if (process.env.minifier_child) {
 }
 
 function executeAction(action, fork, callback) {
-	if (fork) {
+	if (fork && (pool.length - free.length) < Minifier.maxThreads) {
 		forkAction(action, callback);
 	} else {
 		if (typeof actions[action.act] !== 'function') {
@@ -153,32 +168,38 @@ function concat(data, callback) {
 actions.concat = concat;
 
 function minifyJS(data, callback) {
-	var minified;
+	if (data.batch) {
+		async.eachLimit(data.files, 1000, function (ref, next) {
+			var srcPath = ref.srcPath;
+			var destPath = ref.destPath;
+
+			fs.readFile(srcPath, function (err, buffer) {
+				if (err && err.code === 'ENOENT') {
+					return next(null, null);
+				}
+				if (err) {
+					return next(err);
+				}
 
-	if (data.fromSource) {
+				try {
-		var sources = data.source;
+					var minified = uglifyjs.minify(buffer.toString(), {
-		var multiple = Array.isArray(sources);
+						// outSourceMap: data.filename + '.map',
-		if (!multiple) {
+						compress: data.compress,
-			sources = [sources];
+						fromString: true,
-		}
+						output: {
+							// suppress uglify line length warnings
+							max_line_len: 400000,
+						},
+					});
 
-		try {
+					fs.writeFile(destPath, minified.code, next);
-			minified = sources.map(function (source) {
+				} catch (e) {
-				return uglifyjs.minify(source, {
+					next(e);
-					// outSourceMap: data.filename + '.map',
+				}
-					compress: data.compress,
-					fromString: true,
-					output: {
-						// suppress uglify line length warnings
-						max_line_len: 400000,
-					},
-				});
 			});
-		} catch (e) {
+		}, callback);
-			return callback(e);
-		}
 
-		return callback(null, multiple ? minified : minified[0]);
+		return;
 	}
 
 	if (data.files && data.files.length) {
@@ -188,16 +209,16 @@ function minifyJS(data, callback) {
 			}
 
 			try {
-				minified = uglifyjs.minify(scripts, {
+				var minified = uglifyjs.minify(scripts, {
 					// outSourceMap: data.filename + '.map',
 					compress: data.compress,
 					fromString: false,
 				});
+
+				callback(null, minified);
 			} catch (e) {
-				return callback(e);
+				callback(e);
 			}
-
-			callback(null, minified);
 		});
 
 		return;
@@ -216,11 +237,11 @@ Minifier.js.bundle = function (scripts, minify, fork, callback) {
 	}, fork, callback);
 };
 
-Minifier.js.minify = function (source, fork, callback) {
+Minifier.js.minifyBatch = function (scripts, fork, callback) {
 	executeAction({
 		act: 'minifyJS',
-		fromSource: true,
+		files: scripts,
-		source: source,
+		batch: true,
 	}, fork, callback);
 };