From e8caee3c4c46125b628563181145c9c1d25c07cb Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 18 May 2017 17:50:49 -0600 Subject: [PATCH 1/3] Restrict total threads So machines with a small amount of cores build faster --- src/meta/build.js | 3 +-- src/meta/minifier.js | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/meta/build.js b/src/meta/build.js index e88bbb17e9..cc3321923f 100644 --- a/src/meta/build.js +++ b/src/meta/build.js @@ -2,7 +2,6 @@ var async = require('async'); var winston = require('winston'); -var os = require('os'); var nconf = require('nconf'); var padstart = require('lodash.padstart'); @@ -181,7 +180,7 @@ function build(targets, callback) { async.series([ beforeBuild, function (next) { - var parallel = os.cpus().length > 1 && !nconf.get('series'); + var parallel = !nconf.get('series'); if (parallel) { winston.info('[build] Building in parallel mode'); } else { diff --git a/src/meta/minifier.js b/src/meta/minifier.js index b1f2888b16..53ca9f5f28 100644 --- a/src/meta/minifier.js +++ b/src/meta/minifier.js @@ -5,6 +5,7 @@ var async = require('async'); var fs = require('fs'); var childProcess = require('child_process'); var os = require('os'); +var winston = require('winston'); var less = require('less'); var postcss = require('postcss'); var autoprefixer = require('autoprefixer'); @@ -39,6 +40,10 @@ function setupDebugging() { var children = []; +Minifier.maxThreads = os.cpus().length - 1; + +winston.verbose('[minifier] utilizing a maximum of ' + Minifier.maxThreads + ' additional threads'); + Minifier.killAll = function () { children.forEach(function (child) { child.kill('SIGTERM'); @@ -65,13 +70,14 @@ function forkAction(action, callback) { children.push(proc); proc.on('message', function (message) { + proc.kill(); + removeChild(proc); + if (message.type === 'error') { - proc.kill(); - return callback(new Error(message.message)); + return callback(message.err); } if (message.type === 'end') { - proc.kill(); callback(null, message.result); } }); @@ -85,10 +91,6 @@ function forkAction(action, callback) { type: 'action', action: action, }); - - proc.on('close', function () { - removeChild(proc); - }); } var actions = {}; @@ -109,7 +111,7 @@ if (process.env.minifier_child) { if (err) { process.send({ type: 'error', - message: err.message, + err: err, }); return; } @@ -124,7 +126,7 @@ if (process.env.minifier_child) { } function executeAction(action, fork, callback) { - if (fork) { + if (fork && children.length < Minifier.maxThreads) { forkAction(action, callback); } else { if (typeof actions[action.act] !== 'function') { From 9f5ce24993ff6690191c1a2c015d5c0e024498a7 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 18 May 2017 21:20:04 -0600 Subject: [PATCH 2/3] Minify modules in a batch --- src/meta/js.js | 76 ++++++++++++++++++------------- src/meta/minifier.js | 105 +++++++++++++++++++++++++------------------ 2 files changed, 106 insertions(+), 75 deletions(-) diff --git a/src/meta/js.js b/src/meta/js.js index e7b22939fc..bfae9260c2 100644 --- a/src/meta/js.js +++ b/src/meta/js.js @@ -88,44 +88,56 @@ module.exports = function (Meta) { }, }; - function minifyModules(modules, fork, callback) { - // for it to never fork - // otherwise it spawns way too many processes - // maybe eventually we can pool modules - // and pass the pools to the minifer - // to reduce the total number of threads - fork = false; + function copyFile(source, target, cb) { + var called = false; - async.eachLimit(modules, 500, function (mod, next) { - var srcPath = mod.srcPath; - var destPath = mod.destPath; + var rd = fs.createReadStream(source); + rd.on('error', done); - async.parallel({ - dirped: function (cb) { - mkdirp(path.dirname(destPath), cb); - }, - minified: function (cb) { - fs.readFile(srcPath, function (err, buffer) { - if (err) { - return cb(err); - } + var wr = fs.createWriteStream(target); + wr.on('error', done); + wr.on('close', function () { + done(); + }); + rd.pipe(wr); - if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) { - return cb(null, { code: buffer.toString() }); - } + function done(err) { + if (!called) { + cb(err); + called = true; + } + } + } - minifier.js.minify(buffer.toString(), fork, cb); - }); - }, - }, function (err, results) { - if (err) { - return next(err); + function minifyModules(modules, fork, callback) { + async.eachLimit(modules, 1000, function (mod, next) { + mkdirp(path.dirname(mod.destPath), next); + }, function (err) { + if (err) { + return callback(err); + } + + var filtered = modules.reduce(function (prev, mod) { + if (mod.srcPath.endsWith('.min.js') || path.dirname(mod.srcPath).endsWith('min')) { + prev.skip.push(mod); + } else { + prev.minify.push(mod); } - var minified = results.minified; - fs.writeFile(destPath, minified.code, next); - }); - }, callback); + return prev; + }, { minify: [], skip: [] }); + + async.parallel([ + function (cb) { + minifier.js.minifyBatch(filtered.minify, fork, cb); + }, + function (cb) { + async.eachLimit(filtered.skip, 500, function (mod, next) { + copyFile(mod.srcPath, mod.destPath, next); + }, cb); + }, + ], callback); + }); } function linkModules(callback) { diff --git a/src/meta/minifier.js b/src/meta/minifier.js index 53ca9f5f28..da7c570ce7 100644 --- a/src/meta/minifier.js +++ b/src/meta/minifier.js @@ -38,27 +38,26 @@ function setupDebugging() { return forkProcessParams; } -var children = []; +var pool = []; +var free = []; Minifier.maxThreads = os.cpus().length - 1; winston.verbose('[minifier] utilizing a maximum of ' + Minifier.maxThreads + ' additional threads'); Minifier.killAll = function () { - children.forEach(function (child) { + pool.forEach(function (child) { child.kill('SIGTERM'); }); - children = []; + pool.length = 0; }; -function removeChild(proc) { - children = children.filter(function (child) { - return child !== proc; - }); -} +function getChild() { + if (free.length) { + return free.shift(); + } -function forkAction(action, callback) { var forkProcessParams = setupDebugging(); var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, { cwd: __dirname, @@ -66,12 +65,26 @@ function forkAction(action, callback) { minifier_child: true, }, })); + pool.push(proc); + + return proc; +} - children.push(proc); +function freeChild(proc) { + proc.removeAllListeners(); + free.push(proc); +} + +function removeChild(proc) { + var i = pool.indexOf(proc); + pool.splice(i, 1); +} + +function forkAction(action, callback) { + var proc = getChild(); proc.on('message', function (message) { - proc.kill(); - removeChild(proc); + freeChild(proc); if (message.type === 'error') { return callback(message.err); @@ -102,7 +115,7 @@ if (process.env.minifier_child) { if (typeof actions[action.act] !== 'function') { process.send({ type: 'error', - message: 'Unknown action', + err: Error('Unknown action'), }); return; } @@ -126,7 +139,7 @@ if (process.env.minifier_child) { } function executeAction(action, fork, callback) { - if (fork && children.length < Minifier.maxThreads) { + if (fork && (pool.length - free.length) < Minifier.maxThreads) { forkAction(action, callback); } else { if (typeof actions[action.act] !== 'function') { @@ -155,32 +168,38 @@ function concat(data, callback) { actions.concat = concat; function minifyJS(data, callback) { - var minified; + if (data.batch) { + async.eachLimit(data.files, 1000, function (ref, next) { + var srcPath = ref.srcPath; + var destPath = ref.destPath; + + fs.readFile(srcPath, function (err, buffer) { + if (err && err.code === 'ENOENT') { + return next(null, null); + } + if (err) { + return next(err); + } - if (data.fromSource) { - var sources = data.source; - var multiple = Array.isArray(sources); - if (!multiple) { - sources = [sources]; - } + try { + var minified = uglifyjs.minify(buffer.toString(), { + // outSourceMap: data.filename + '.map', + compress: data.compress, + fromString: true, + output: { + // suppress uglify line length warnings + max_line_len: 400000, + }, + }); - try { - minified = sources.map(function (source) { - return uglifyjs.minify(source, { - // outSourceMap: data.filename + '.map', - compress: data.compress, - fromString: true, - output: { - // suppress uglify line length warnings - max_line_len: 400000, - }, - }); + fs.writeFile(destPath, minified.code, next); + } catch (e) { + next(e); + } }); - } catch (e) { - return callback(e); - } + }, callback); - return callback(null, multiple ? minified : minified[0]); + return; } if (data.files && data.files.length) { @@ -190,16 +209,16 @@ function minifyJS(data, callback) { } try { - minified = uglifyjs.minify(scripts, { + var minified = uglifyjs.minify(scripts, { // outSourceMap: data.filename + '.map', compress: data.compress, fromString: false, }); + + callback(null, minified); } catch (e) { - return callback(e); + callback(e); } - - callback(null, minified); }); return; @@ -218,11 +237,11 @@ Minifier.js.bundle = function (scripts, minify, fork, callback) { }, fork, callback); }; -Minifier.js.minify = function (source, fork, callback) { +Minifier.js.minifyBatch = function (scripts, fork, callback) { executeAction({ act: 'minifyJS', - fromSource: true, - source: source, + files: scripts, + batch: true, }, fork, callback); }; From 4c1e25c8ce752bc3130b0ac83c849e568d070cd4 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Fri, 19 May 2017 14:27:52 -0600 Subject: [PATCH 3/3] Link instead of copying files Only mkdirp the necessary directories --- src/meta/js.js | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/src/meta/js.js b/src/meta/js.js index bfae9260c2..d0399c70c1 100644 --- a/src/meta/js.js +++ b/src/meta/js.js @@ -88,31 +88,16 @@ module.exports = function (Meta) { }, }; - function copyFile(source, target, cb) { - var called = false; - - var rd = fs.createReadStream(source); - rd.on('error', done); - - var wr = fs.createWriteStream(target); - wr.on('error', done); - wr.on('close', function () { - done(); - }); - rd.pipe(wr); - - function done(err) { - if (!called) { - cb(err); - called = true; + function minifyModules(modules, fork, callback) { + var moduleDirs = modules.reduce(function (prev, mod) { + var dir = path.resolve(path.dirname(mod.destPath)); + if (prev.indexOf(dir) === -1) { + prev.push(dir); } - } - } + return prev; + }, []); - function minifyModules(modules, fork, callback) { - async.eachLimit(modules, 1000, function (mod, next) { - mkdirp(path.dirname(mod.destPath), next); - }, function (err) { + async.eachLimit(moduleDirs, 1000, mkdirp, function (err) { if (err) { return callback(err); } @@ -133,7 +118,7 @@ module.exports = function (Meta) { }, function (cb) { async.eachLimit(filtered.skip, 500, function (mod, next) { - copyFile(mod.srcPath, mod.destPath, next); + file.link(mod.srcPath, mod.destPath, next); }, cb); }, ], callback);