hyperzlib
/
nodebb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Minify modules in a batch

Browse Source
v1.18.x
Peter Jaszkowiak 8 years ago
parent e8caee3c4c
commit 9f5ce24993
2 changed files with 106 additions and 75 deletions
Show Stats Download Patch File Download Diff File

76
src/meta/js.js
Unescape Escape View File

@ -88,44 +88,56 @@ module.exports = function (Meta) {
},
};
function minifyModules(modules, fork, callback) {
// for it to never fork
// otherwise it spawns way too many processes
// maybe eventually we can pool modules
// and pass the pools to the minifer
// to reduce the total number of threads
fork = false;
function copyFile(source, target, cb) {
var called = false;
async.eachLimit(modules, 500, function (mod, next) {
var srcPath = mod.srcPath;
var destPath = mod.destPath;
var rd = fs.createReadStream(source);
rd.on('error', done);
async.parallel({
dirped: function (cb) {
mkdirp(path.dirname(destPath), cb);
},
minified: function (cb) {
fs.readFile(srcPath, function (err, buffer) {
if (err) {
return cb(err);
}
var wr = fs.createWriteStream(target);
wr.on('error', done);
wr.on('close', function () {
done();
});
rd.pipe(wr);
if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) {
return cb(null, { code: buffer.toString() });
}
function done(err) {
if (!called) {
cb(err);
called = true;
}
}
}
minifier.js.minify(buffer.toString(), fork, cb);
});
},
}, function (err, results) {
if (err) {
return next(err);
function minifyModules(modules, fork, callback) {
async.eachLimit(modules, 1000, function (mod, next) {
mkdirp(path.dirname(mod.destPath), next);
}, function (err) {
if (err) {
return callback(err);
}
var filtered = modules.reduce(function (prev, mod) {
if (mod.srcPath.endsWith('.min.js') || path.dirname(mod.srcPath).endsWith('min')) {
prev.skip.push(mod);
} else {
prev.minify.push(mod);
}
var minified = results.minified;
fs.writeFile(destPath, minified.code, next);
});
}, callback);
return prev;
}, { minify: [], skip: [] });
async.parallel([
function (cb) {
minifier.js.minifyBatch(filtered.minify, fork, cb);
},
function (cb) {
async.eachLimit(filtered.skip, 500, function (mod, next) {
copyFile(mod.srcPath, mod.destPath, next);
}, cb);
},
], callback);
});
}
function linkModules(callback) {

105
src/meta/minifier.js
Unescape Escape View File

@ -38,27 +38,26 @@ function setupDebugging() {
return forkProcessParams;
}
var children = [];
var pool = [];
var free = [];
Minifier.maxThreads = os.cpus().length - 1;
winston.verbose('[minifier] utilizing a maximum of ' + Minifier.maxThreads + ' additional threads');
Minifier.killAll = function () {
children.forEach(function (child) {
pool.forEach(function (child) {
child.kill('SIGTERM');
});
children = [];
pool.length = 0;
};
function removeChild(proc) {
children = children.filter(function (child) {
return child !== proc;
});
}
function getChild() {
if (free.length) {
return free.shift();
}
function forkAction(action, callback) {
var forkProcessParams = setupDebugging();
var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, {
cwd: __dirname,
@ -66,12 +65,26 @@ function forkAction(action, callback) {
minifier_child: true,
},
}));
pool.push(proc);
return proc;
}
children.push(proc);
function freeChild(proc) {
proc.removeAllListeners();
free.push(proc);
}
function removeChild(proc) {
var i = pool.indexOf(proc);
pool.splice(i, 1);
}
function forkAction(action, callback) {
var proc = getChild();
proc.on('message', function (message) {
proc.kill();
removeChild(proc);
freeChild(proc);
if (message.type === 'error') {
return callback(message.err);
@ -102,7 +115,7 @@ if (process.env.minifier_child) {
if (typeof actions[action.act] !== 'function') {
process.send({
type: 'error',
message: 'Unknown action',
err: Error('Unknown action'),
});
return;
}
@ -126,7 +139,7 @@ if (process.env.minifier_child) {
}
function executeAction(action, fork, callback) {
if (fork && children.length < Minifier.maxThreads) {
if (fork && (pool.length - free.length) < Minifier.maxThreads) {
forkAction(action, callback);
} else {
if (typeof actions[action.act] !== 'function') {
@ -155,32 +168,38 @@ function concat(data, callback) {
actions.concat = concat;
function minifyJS(data, callback) {
var minified;
if (data.batch) {
async.eachLimit(data.files, 1000, function (ref, next) {
var srcPath = ref.srcPath;
var destPath = ref.destPath;
fs.readFile(srcPath, function (err, buffer) {
if (err && err.code === 'ENOENT') {
return next(null, null);
}
if (err) {
return next(err);
}
if (data.fromSource) {
var sources = data.source;
var multiple = Array.isArray(sources);
if (!multiple) {
sources = [sources];
}
try {
var minified = uglifyjs.minify(buffer.toString(), {
// outSourceMap: data.filename + '.map',
compress: data.compress,
fromString: true,
output: {
// suppress uglify line length warnings
max_line_len: 400000,
},
});
try {
minified = sources.map(function (source) {
return uglifyjs.minify(source, {
// outSourceMap: data.filename + '.map',
compress: data.compress,
fromString: true,
output: {
// suppress uglify line length warnings
max_line_len: 400000,
},
});
fs.writeFile(destPath, minified.code, next);
} catch (e) {
next(e);
}
});
} catch (e) {
return callback(e);
}
}, callback);
return callback(null, multiple ? minified : minified[0]);
return;
}
if (data.files && data.files.length) {
@ -190,16 +209,16 @@ function minifyJS(data, callback) {
}
try {
minified = uglifyjs.minify(scripts, {
var minified = uglifyjs.minify(scripts, {
// outSourceMap: data.filename + '.map',
compress: data.compress,
fromString: false,
});
callback(null, minified);
} catch (e) {
return callback(e);
callback(e);
}
callback(null, minified);
});
return;
@ -218,11 +237,11 @@ Minifier.js.bundle = function (scripts, minify, fork, callback) {
}, fork, callback);
};
Minifier.js.minify = function (source, fork, callback) {
Minifier.js.minifyBatch = function (scripts, fork, callback) {
executeAction({
act: 'minifyJS',
fromSource: true,
source: source,
files: scripts,
batch: true,
}, fork, callback);
};

Write Preview
Loading…
Cancel
Save