From 9f5ce24993ff6690191c1a2c015d5c0e024498a7 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 18 May 2017 21:20:04 -0600 Subject: [PATCH] Minify modules in a batch --- src/meta/js.js | 76 ++++++++++++++++++------------- src/meta/minifier.js | 105 +++++++++++++++++++++++++------------------ 2 files changed, 106 insertions(+), 75 deletions(-) diff --git a/src/meta/js.js b/src/meta/js.js index e7b22939fc..bfae9260c2 100644 --- a/src/meta/js.js +++ b/src/meta/js.js @@ -88,44 +88,56 @@ module.exports = function (Meta) { }, }; - function minifyModules(modules, fork, callback) { - // for it to never fork - // otherwise it spawns way too many processes - // maybe eventually we can pool modules - // and pass the pools to the minifer - // to reduce the total number of threads - fork = false; + function copyFile(source, target, cb) { + var called = false; - async.eachLimit(modules, 500, function (mod, next) { - var srcPath = mod.srcPath; - var destPath = mod.destPath; + var rd = fs.createReadStream(source); + rd.on('error', done); - async.parallel({ - dirped: function (cb) { - mkdirp(path.dirname(destPath), cb); - }, - minified: function (cb) { - fs.readFile(srcPath, function (err, buffer) { - if (err) { - return cb(err); - } + var wr = fs.createWriteStream(target); + wr.on('error', done); + wr.on('close', function () { + done(); + }); + rd.pipe(wr); - if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) { - return cb(null, { code: buffer.toString() }); - } + function done(err) { + if (!called) { + cb(err); + called = true; + } + } + } - minifier.js.minify(buffer.toString(), fork, cb); - }); - }, - }, function (err, results) { - if (err) { - return next(err); + function minifyModules(modules, fork, callback) { + async.eachLimit(modules, 1000, function (mod, next) { + mkdirp(path.dirname(mod.destPath), next); + }, function (err) { + if (err) { + return callback(err); + } + + var filtered = modules.reduce(function (prev, mod) { + if (mod.srcPath.endsWith('.min.js') || path.dirname(mod.srcPath).endsWith('min')) { + prev.skip.push(mod); + } else { + prev.minify.push(mod); } - var minified = results.minified; - fs.writeFile(destPath, minified.code, next); - }); - }, callback); + return prev; + }, { minify: [], skip: [] }); + + async.parallel([ + function (cb) { + minifier.js.minifyBatch(filtered.minify, fork, cb); + }, + function (cb) { + async.eachLimit(filtered.skip, 500, function (mod, next) { + copyFile(mod.srcPath, mod.destPath, next); + }, cb); + }, + ], callback); + }); } function linkModules(callback) { diff --git a/src/meta/minifier.js b/src/meta/minifier.js index 53ca9f5f28..da7c570ce7 100644 --- a/src/meta/minifier.js +++ b/src/meta/minifier.js @@ -38,27 +38,26 @@ function setupDebugging() { return forkProcessParams; } -var children = []; +var pool = []; +var free = []; Minifier.maxThreads = os.cpus().length - 1; winston.verbose('[minifier] utilizing a maximum of ' + Minifier.maxThreads + ' additional threads'); Minifier.killAll = function () { - children.forEach(function (child) { + pool.forEach(function (child) { child.kill('SIGTERM'); }); - children = []; + pool.length = 0; }; -function removeChild(proc) { - children = children.filter(function (child) { - return child !== proc; - }); -} +function getChild() { + if (free.length) { + return free.shift(); + } -function forkAction(action, callback) { var forkProcessParams = setupDebugging(); var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, { cwd: __dirname, @@ -66,12 +65,26 @@ function forkAction(action, callback) { minifier_child: true, }, })); + pool.push(proc); + + return proc; +} - children.push(proc); +function freeChild(proc) { + proc.removeAllListeners(); + free.push(proc); +} + +function removeChild(proc) { + var i = pool.indexOf(proc); + pool.splice(i, 1); +} + +function forkAction(action, callback) { + var proc = getChild(); proc.on('message', function (message) { - proc.kill(); - removeChild(proc); + freeChild(proc); if (message.type === 'error') { return callback(message.err); @@ -102,7 +115,7 @@ if (process.env.minifier_child) { if (typeof actions[action.act] !== 'function') { process.send({ type: 'error', - message: 'Unknown action', + err: Error('Unknown action'), }); return; } @@ -126,7 +139,7 @@ if (process.env.minifier_child) { } function executeAction(action, fork, callback) { - if (fork && children.length < Minifier.maxThreads) { + if (fork && (pool.length - free.length) < Minifier.maxThreads) { forkAction(action, callback); } else { if (typeof actions[action.act] !== 'function') { @@ -155,32 +168,38 @@ function concat(data, callback) { actions.concat = concat; function minifyJS(data, callback) { - var minified; + if (data.batch) { + async.eachLimit(data.files, 1000, function (ref, next) { + var srcPath = ref.srcPath; + var destPath = ref.destPath; + + fs.readFile(srcPath, function (err, buffer) { + if (err && err.code === 'ENOENT') { + return next(null, null); + } + if (err) { + return next(err); + } - if (data.fromSource) { - var sources = data.source; - var multiple = Array.isArray(sources); - if (!multiple) { - sources = [sources]; - } + try { + var minified = uglifyjs.minify(buffer.toString(), { + // outSourceMap: data.filename + '.map', + compress: data.compress, + fromString: true, + output: { + // suppress uglify line length warnings + max_line_len: 400000, + }, + }); - try { - minified = sources.map(function (source) { - return uglifyjs.minify(source, { - // outSourceMap: data.filename + '.map', - compress: data.compress, - fromString: true, - output: { - // suppress uglify line length warnings - max_line_len: 400000, - }, - }); + fs.writeFile(destPath, minified.code, next); + } catch (e) { + next(e); + } }); - } catch (e) { - return callback(e); - } + }, callback); - return callback(null, multiple ? minified : minified[0]); + return; } if (data.files && data.files.length) { @@ -190,16 +209,16 @@ function minifyJS(data, callback) { } try { - minified = uglifyjs.minify(scripts, { + var minified = uglifyjs.minify(scripts, { // outSourceMap: data.filename + '.map', compress: data.compress, fromString: false, }); + + callback(null, minified); } catch (e) { - return callback(e); + callback(e); } - - callback(null, minified); }); return; @@ -218,11 +237,11 @@ Minifier.js.bundle = function (scripts, minify, fork, callback) { }, fork, callback); }; -Minifier.js.minify = function (source, fork, callback) { +Minifier.js.minifyBatch = function (scripts, fork, callback) { executeAction({ act: 'minifyJS', - fromSource: true, - source: source, + files: scripts, + batch: true, }, fork, callback); };