hyperzlib
/
nodebb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5690 from NodeBB/build-refactor

Browse Source 
Restrict total threads, minify modules in a batch, link instead of copying pre-minified files
v1.18.x
Barış Soner Uşaklı 8 years ago committed by GitHub
parent 12e3369522 4c1e25c8ce
commit 9a9d1cf3f4
3 changed files with 100 additions and 83 deletions
Show Stats Download Patch File Download Diff File

3
src/meta/build.js
Unescape Escape View File

@ -2,7 +2,6 @@
var async = require('async');
var winston = require('winston');
var os = require('os');
var nconf = require('nconf');
var padstart = require('lodash.padstart');
@ -181,7 +180,7 @@ function build(targets, callback) {
async.series([
beforeBuild,
function (next) {
var parallel = os.cpus().length > 1 && !nconf.get('series');
var parallel = !nconf.get('series');
if (parallel) {
winston.info('[build] Building in parallel mode');
} else {

61
src/meta/js.js
Unescape Escape View File

@ -89,43 +89,40 @@ module.exports = function (Meta) {
};
function minifyModules(modules, fork, callback) {
// for it to never fork
// otherwise it spawns way too many processes
// maybe eventually we can pool modules
// and pass the pools to the minifer
// to reduce the total number of threads
fork = false;
var moduleDirs = modules.reduce(function (prev, mod) {
var dir = path.resolve(path.dirname(mod.destPath));
if (prev.indexOf(dir) === -1) {
prev.push(dir);
}
return prev;
}, []);
async.eachLimit(modules, 500, function (mod, next) {
var srcPath = mod.srcPath;
var destPath = mod.destPath;
async.eachLimit(moduleDirs, 1000, mkdirp, function (err) {
if (err) {
return callback(err);
}
async.parallel({
dirped: function (cb) {
mkdirp(path.dirname(destPath), cb);
},
minified: function (cb) {
fs.readFile(srcPath, function (err, buffer) {
if (err) {
return cb(err);
}
var filtered = modules.reduce(function (prev, mod) {
if (mod.srcPath.endsWith('.min.js') || path.dirname(mod.srcPath).endsWith('min')) {
prev.skip.push(mod);
} else {
prev.minify.push(mod);
}
if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) {
return cb(null, { code: buffer.toString() });
}
return prev;
}, { minify: [], skip: [] });
minifier.js.minify(buffer.toString(), fork, cb);
});
async.parallel([
function (cb) {
minifier.js.minifyBatch(filtered.minify, fork, cb);
},
}, function (err, results) {
if (err) {
return next(err);
}
var minified = results.minified;
fs.writeFile(destPath, minified.code, next);
});
}, callback);
function (cb) {
async.eachLimit(filtered.skip, 500, function (mod, next) {
file.link(mod.srcPath, mod.destPath, next);
}, cb);
},
], callback);
});
}
function linkModules(callback) {

119
src/meta/minifier.js
Unescape Escape View File

@ -5,6 +5,7 @@ var async = require('async');
var fs = require('fs');
var childProcess = require('child_process');
var os = require('os');
var winston = require('winston');
var less = require('less');
var postcss = require('postcss');
var autoprefixer = require('autoprefixer');
@ -37,23 +38,26 @@ function setupDebugging() {
return forkProcessParams;
}
var children = [];
var pool = [];
var free = [];
Minifier.maxThreads = os.cpus().length - 1;
winston.verbose('[minifier] utilizing a maximum of ' + Minifier.maxThreads + ' additional threads');
Minifier.killAll = function () {
children.forEach(function (child) {
pool.forEach(function (child) {
child.kill('SIGTERM');
});
children = [];
pool.length = 0;
};
function removeChild(proc) {
children = children.filter(function (child) {
return child !== proc;
});
}
function getChild() {
if (free.length) {
return free.shift();
}
function forkAction(action, callback) {
var forkProcessParams = setupDebugging();
var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, {
cwd: __dirname,
@ -61,17 +65,32 @@ function forkAction(action, callback) {
minifier_child: true,
},
}));
pool.push(proc);
children.push(proc);
return proc;
}
function freeChild(proc) {
proc.removeAllListeners();
free.push(proc);
}
function removeChild(proc) {
var i = pool.indexOf(proc);
pool.splice(i, 1);
}
function forkAction(action, callback) {
var proc = getChild();
proc.on('message', function (message) {
freeChild(proc);
if (message.type === 'error') {
proc.kill();
return callback(new Error(message.message));
return callback(message.err);
}
if (message.type === 'end') {
proc.kill();
callback(null, message.result);
}
});
@ -85,10 +104,6 @@ function forkAction(action, callback) {
type: 'action',
action: action,
});
proc.on('close', function () {
removeChild(proc);
});
}
var actions = {};
@ -100,7 +115,7 @@ if (process.env.minifier_child) {
if (typeof actions[action.act] !== 'function') {
process.send({
type: 'error',
message: 'Unknown action',
err: Error('Unknown action'),
});
return;
}
@ -109,7 +124,7 @@ if (process.env.minifier_child) {
if (err) {
process.send({
type: 'error',
message: err.message,
err: err,
});
return;
}
@ -124,7 +139,7 @@ if (process.env.minifier_child) {
}
function executeAction(action, fork, callback) {
if (fork) {
if (fork && (pool.length - free.length) < Minifier.maxThreads) {
forkAction(action, callback);
} else {
if (typeof actions[action.act] !== 'function') {
@ -153,32 +168,38 @@ function concat(data, callback) {
actions.concat = concat;
function minifyJS(data, callback) {
var minified;
if (data.batch) {
async.eachLimit(data.files, 1000, function (ref, next) {
var srcPath = ref.srcPath;
var destPath = ref.destPath;
fs.readFile(srcPath, function (err, buffer) {
if (err && err.code === 'ENOENT') {
return next(null, null);
}
if (err) {
return next(err);
}
if (data.fromSource) {
var sources = data.source;
var multiple = Array.isArray(sources);
if (!multiple) {
sources = [sources];
}
try {
var minified = uglifyjs.minify(buffer.toString(), {
// outSourceMap: data.filename + '.map',
compress: data.compress,
fromString: true,
output: {
// suppress uglify line length warnings
max_line_len: 400000,
},
});
try {
minified = sources.map(function (source) {
return uglifyjs.minify(source, {
// outSourceMap: data.filename + '.map',
compress: data.compress,
fromString: true,
output: {
// suppress uglify line length warnings
max_line_len: 400000,
},
});
fs.writeFile(destPath, minified.code, next);
} catch (e) {
next(e);
}
});
} catch (e) {
return callback(e);
}
}, callback);
return callback(null, multiple ? minified : minified[0]);
return;
}
if (data.files && data.files.length) {
@ -188,16 +209,16 @@ function minifyJS(data, callback) {
}
try {
minified = uglifyjs.minify(scripts, {
var minified = uglifyjs.minify(scripts, {
// outSourceMap: data.filename + '.map',
compress: data.compress,
fromString: false,
});
callback(null, minified);
} catch (e) {
return callback(e);
callback(e);
}
callback(null, minified);
});
return;
@ -216,11 +237,11 @@ Minifier.js.bundle = function (scripts, minify, fork, callback) {
}, fork, callback);
};
Minifier.js.minify = function (source, fork, callback) {
Minifier.js.minifyBatch = function (scripts, fork, callback) {
executeAction({
act: 'minifyJS',
fromSource: true,
source: source,
files: scripts,
batch: true,
}, fork, callback);
};

Write Preview
Loading…
Cancel
Save