diff --git a/src/meta/build.js b/src/meta/build.js index e88bbb17e9..cc3321923f 100644 --- a/src/meta/build.js +++ b/src/meta/build.js @@ -2,7 +2,6 @@ var async = require('async'); var winston = require('winston'); -var os = require('os'); var nconf = require('nconf'); var padstart = require('lodash.padstart'); @@ -181,7 +180,7 @@ function build(targets, callback) { async.series([ beforeBuild, function (next) { - var parallel = os.cpus().length > 1 && !nconf.get('series'); + var parallel = !nconf.get('series'); if (parallel) { winston.info('[build] Building in parallel mode'); } else { diff --git a/src/meta/js.js b/src/meta/js.js index e7b22939fc..d0399c70c1 100644 --- a/src/meta/js.js +++ b/src/meta/js.js @@ -89,43 +89,40 @@ module.exports = function (Meta) { }; function minifyModules(modules, fork, callback) { - // for it to never fork - // otherwise it spawns way too many processes - // maybe eventually we can pool modules - // and pass the pools to the minifer - // to reduce the total number of threads - fork = false; + var moduleDirs = modules.reduce(function (prev, mod) { + var dir = path.resolve(path.dirname(mod.destPath)); + if (prev.indexOf(dir) === -1) { + prev.push(dir); + } + return prev; + }, []); - async.eachLimit(modules, 500, function (mod, next) { - var srcPath = mod.srcPath; - var destPath = mod.destPath; + async.eachLimit(moduleDirs, 1000, mkdirp, function (err) { + if (err) { + return callback(err); + } - async.parallel({ - dirped: function (cb) { - mkdirp(path.dirname(destPath), cb); - }, - minified: function (cb) { - fs.readFile(srcPath, function (err, buffer) { - if (err) { - return cb(err); - } + var filtered = modules.reduce(function (prev, mod) { + if (mod.srcPath.endsWith('.min.js') || path.dirname(mod.srcPath).endsWith('min')) { + prev.skip.push(mod); + } else { + prev.minify.push(mod); + } - if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) { - return cb(null, { code: buffer.toString() }); - } + return prev; + }, { minify: [], skip: [] }); - minifier.js.minify(buffer.toString(), fork, cb); - }); + async.parallel([ + function (cb) { + minifier.js.minifyBatch(filtered.minify, fork, cb); }, - }, function (err, results) { - if (err) { - return next(err); - } - - var minified = results.minified; - fs.writeFile(destPath, minified.code, next); - }); - }, callback); + function (cb) { + async.eachLimit(filtered.skip, 500, function (mod, next) { + file.link(mod.srcPath, mod.destPath, next); + }, cb); + }, + ], callback); + }); } function linkModules(callback) { diff --git a/src/meta/minifier.js b/src/meta/minifier.js index b1f2888b16..da7c570ce7 100644 --- a/src/meta/minifier.js +++ b/src/meta/minifier.js @@ -5,6 +5,7 @@ var async = require('async'); var fs = require('fs'); var childProcess = require('child_process'); var os = require('os'); +var winston = require('winston'); var less = require('less'); var postcss = require('postcss'); var autoprefixer = require('autoprefixer'); @@ -37,23 +38,26 @@ function setupDebugging() { return forkProcessParams; } -var children = []; +var pool = []; +var free = []; + +Minifier.maxThreads = os.cpus().length - 1; + +winston.verbose('[minifier] utilizing a maximum of ' + Minifier.maxThreads + ' additional threads'); Minifier.killAll = function () { - children.forEach(function (child) { + pool.forEach(function (child) { child.kill('SIGTERM'); }); - children = []; + pool.length = 0; }; -function removeChild(proc) { - children = children.filter(function (child) { - return child !== proc; - }); -} +function getChild() { + if (free.length) { + return free.shift(); + } -function forkAction(action, callback) { var forkProcessParams = setupDebugging(); var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, { cwd: __dirname, @@ -61,17 +65,32 @@ function forkAction(action, callback) { minifier_child: true, }, })); + pool.push(proc); - children.push(proc); + return proc; +} + +function freeChild(proc) { + proc.removeAllListeners(); + free.push(proc); +} + +function removeChild(proc) { + var i = pool.indexOf(proc); + pool.splice(i, 1); +} + +function forkAction(action, callback) { + var proc = getChild(); proc.on('message', function (message) { + freeChild(proc); + if (message.type === 'error') { - proc.kill(); - return callback(new Error(message.message)); + return callback(message.err); } if (message.type === 'end') { - proc.kill(); callback(null, message.result); } }); @@ -85,10 +104,6 @@ function forkAction(action, callback) { type: 'action', action: action, }); - - proc.on('close', function () { - removeChild(proc); - }); } var actions = {}; @@ -100,7 +115,7 @@ if (process.env.minifier_child) { if (typeof actions[action.act] !== 'function') { process.send({ type: 'error', - message: 'Unknown action', + err: Error('Unknown action'), }); return; } @@ -109,7 +124,7 @@ if (process.env.minifier_child) { if (err) { process.send({ type: 'error', - message: err.message, + err: err, }); return; } @@ -124,7 +139,7 @@ if (process.env.minifier_child) { } function executeAction(action, fork, callback) { - if (fork) { + if (fork && (pool.length - free.length) < Minifier.maxThreads) { forkAction(action, callback); } else { if (typeof actions[action.act] !== 'function') { @@ -153,32 +168,38 @@ function concat(data, callback) { actions.concat = concat; function minifyJS(data, callback) { - var minified; + if (data.batch) { + async.eachLimit(data.files, 1000, function (ref, next) { + var srcPath = ref.srcPath; + var destPath = ref.destPath; + + fs.readFile(srcPath, function (err, buffer) { + if (err && err.code === 'ENOENT') { + return next(null, null); + } + if (err) { + return next(err); + } - if (data.fromSource) { - var sources = data.source; - var multiple = Array.isArray(sources); - if (!multiple) { - sources = [sources]; - } + try { + var minified = uglifyjs.minify(buffer.toString(), { + // outSourceMap: data.filename + '.map', + compress: data.compress, + fromString: true, + output: { + // suppress uglify line length warnings + max_line_len: 400000, + }, + }); - try { - minified = sources.map(function (source) { - return uglifyjs.minify(source, { - // outSourceMap: data.filename + '.map', - compress: data.compress, - fromString: true, - output: { - // suppress uglify line length warnings - max_line_len: 400000, - }, - }); + fs.writeFile(destPath, minified.code, next); + } catch (e) { + next(e); + } }); - } catch (e) { - return callback(e); - } + }, callback); - return callback(null, multiple ? minified : minified[0]); + return; } if (data.files && data.files.length) { @@ -188,16 +209,16 @@ function minifyJS(data, callback) { } try { - minified = uglifyjs.minify(scripts, { + var minified = uglifyjs.minify(scripts, { // outSourceMap: data.filename + '.map', compress: data.compress, fromString: false, }); + + callback(null, minified); } catch (e) { - return callback(e); + callback(e); } - - callback(null, minified); }); return; @@ -216,11 +237,11 @@ Minifier.js.bundle = function (scripts, minify, fork, callback) { }, fork, callback); }; -Minifier.js.minify = function (source, fork, callback) { +Minifier.js.minifyBatch = function (scripts, fork, callback) { executeAction({ act: 'minifyJS', - fromSource: true, - source: source, + files: scripts, + batch: true, }, fork, callback); };