hyperzlib
/
nodebb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Build refactor

Browse Source 
- Target aliases and bundles
- More concurrency
- Use `--series` to force series processing
- Minifier self-hosted threading
- meta/css delegates to minifier
v1.18.x
Peter Jaszkowiak 8 years ago
parent 11fc79051a
commit 7ccfb5cdc0
6 changed files with 612 additions and 438 deletions
Show Stats Download Patch File Download Diff File

3
package.json
Unescape Escape View File

@ -17,8 +17,8 @@
"coveralls": "istanbul cover _mocha --report lcovonly -- -R dot && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage"
},
"dependencies": {
"async": "2.4.0",
"ace-builds": "^1.2.6",
"async": "2.4.0",
"autoprefixer": "7.0.1",
"bcryptjs": "2.4.3",
"body-parser": "^1.9.0",
@ -45,6 +45,7 @@
"jquery": "^3.1.0",
"json-2-csv": "^2.0.22",
"less": "^2.0.0",
"lodash.padstart": "^4.6.1",
"logrotate-stream": "^0.2.3",
"lru-cache": "4.0.2",
"mime": "^1.3.4",

2
public/less/admin/admin.less
Unescape Escape View File

@ -1,4 +1,4 @@
@import "../../node_modules/bootstrap/less/bootstrap";
@import "bootstrap/less/bootstrap";
@import "./paper/variables";
@import "./paper/bootswatch";
@import "./mixins";

283
src/meta/build.js
Unescape Escape View File

@ -2,149 +2,212 @@
var async = require('async');
var winston = require('winston');
var os = require('os');
var nconf = require('nconf');
var padstart = require('lodash.padstart');
var buildStart;
var cacheBuster = require('./cacheBuster');
var meta;
var valid = ['js', 'clientCSS', 'acpCSS', 'tpl', 'lang', 'sound'];
function step(target, callback) {
var startTime = Date.now();
winston.info('[build] ' + target + ' build started');
exports.buildAll = function (callback) {
exports.build(valid.join(','), callback);
};
return function (err) {
if (err) {
winston.error('[build] ' + target + ' build failed');
return callback(err);
}
exports.build = function build(targets, callback) {
buildStart = Date.now();
var time = (Date.now() - startTime) / 1000;
var db = require('../database');
var meta = require('../meta');
var plugins = require('../plugins');
winston.info('[build] ' + target + ' build completed in ' + time + 'sec');
callback();
};
}
var targetHandlers = {
'plugin static dirs': function (parallel, callback) {
meta.js.linkStatics(callback);
},
'requirejs modules': function (parallel, callback) {
meta.js.buildModules(parallel, callback);
},
'client js bundle': function (parallel, callback) {
meta.js.buildBundle('client', parallel, callback);
},
'admin js bundle': function (parallel, callback) {
meta.js.buildBundle('admin', parallel, callback);
},
javascript: [
'plugin static dirs',
'requirejs modules',
'client js bundle',
'admin js bundle',
],
'client side styles': function (parallel, callback) {
meta.css.buildBundle('client', parallel, callback);
},
'admin control panel styles': function (parallel, callback) {
meta.css.buildBundle('admin', parallel, callback);
},
styles: [
'client side styles',
'admin control panel styles',
],
templates: function (parallel, callback) {
meta.templates.compile(callback);
},
languages: function (parallel, callback) {
meta.languages.build(callback);
},
sounds: function (parallel, callback) {
meta.sounds.build(callback);
},
};
var aliases = {
'plugin static dirs': ['staticdirs'],
'requirejs modules': ['rjs', 'modules'],
'client js bundle': ['clientjs', 'clientscript', 'clientscripts'],
'admin js bundle': ['adminjs', 'adminscript', 'adminscripts'],
javascript: ['js'],
'client side styles': [
'clientcss', 'clientless', 'clientstyles', 'clientstyle',
],
'admin control panel styles': [
'admincss', 'adminless', 'adminstyles', 'adminstyle', 'acpcss', 'acpless', 'acpstyles', 'acpstyle',
],
styles: ['css', 'less', 'style'],
templates: ['tpl'],
languages: ['lang', 'i18n'],
sounds: ['sound'],
};
targets = (targets === true ? valid : targets.split(',').filter(function (target) {
return valid.indexOf(target) !== -1;
}));
aliases = Object.keys(aliases).reduce(function (prev, key) {
var arr = aliases[key];
arr.forEach(function (alias) {
prev[alias] = key;
});
prev[key] = key;
return prev;
}, {});
if (!targets) {
winston.error('[build] No valid build targets found. Aborting.');
return process.exit(0);
}
function beforeBuild(callback) {
var db = require('../database');
var plugins = require('../plugins');
meta = require('../meta');
async.series([
async.apply(db.init),
async.apply(meta.themes.setupPaths),
async.apply(plugins.prepareForBuild),
db.init,
meta.themes.setupPaths,
plugins.prepareForBuild,
], function (err) {
if (err) {
winston.error('[build] Encountered error preparing for build: ' + err.message);
return process.exit(1);
return callback(err);
}
exports.buildTargets(targets, callback);
callback();
});
};
}
exports.buildTargets = function (targets, callback) {
var cacheBuster = require('./cacheBuster');
var meta = require('../meta');
var numCpus = require('os').cpus().length;
var parallel = targets.length > 1 && numCpus > 1;
var allTargets = Object.keys(targetHandlers).filter(function (name) {
return typeof targetHandlers[name] === 'function';
});
function buildTargets(targets, parallel, callback) {
var all = parallel ? async.each : async.eachSeries;
buildStart = buildStart || Date.now();
var length = Math.max.apply(Math, targets.map(function (name) {
return name.length;
}));
var step = function (startTime, target, next, err) {
if (err) {
winston.error('Build failed: ' + err.stack);
process.exit(1);
}
winston.info('[build] ' + target + ' => Completed in ' + ((Date.now() - startTime) / 1000) + 's');
next();
};
all(targets, function (target, next) {
targetHandlers[target](parallel, step(padstart(target, length) + ' ', next));
}, callback);
}
if (parallel) {
winston.verbose('[build] Utilising multiple cores/processes');
} else {
winston.verbose('[build] Utilising single-core');
function build(targets, callback) {
if (targets === true) {
targets = allTargets;
} else if (!Array.isArray(targets)) {
targets = targets.split(',');
}
async[parallel ? 'parallel' : 'series']([
targets = targets
// get full target name
.map(function (target) {
target = target.toLowerCase().replace(/-/g, '');
if (!aliases[target]) {
winston.warn('[build] Unknown target: ' + target);
return false;
}
return aliases[target];
})
// filter nonexistent targets
.filter(Boolean)
// map multitargets to their sets
.reduce(function (prev, target) {
if (Array.isArray(targetHandlers[target])) {
return prev.concat(targetHandlers[target]);
}
return prev.concat(target);
}, [])
// unique
.filter(function (target, i, arr) {
return arr.indexOf(target) === i;
});
if (typeof callback !== 'function') {
callback = function (err) {
if (err) {
winston.error(err);
process.exit(1);
} else {
process.exit(0);
}
};
}
if (!targets) {
winston.info('[build] No valid targets supplied. Aborting.');
callback();
}
var startTime;
var totalTime;
async.series([
beforeBuild,
function (next) {
if (targets.indexOf('js') !== -1) {
winston.info('[build] Building javascript');
var startTime = Date.now();
async.series([
meta.js.buildModules,
meta.js.linkStatics,
async.apply(meta.js.minify, 'nodebb.min.js'),
async.apply(meta.js.minify, 'acp.min.js'),
], step.bind(this, startTime, 'js', next));
var parallel = os.cpus().length > 1 && !nconf.get('series');
if (parallel) {
winston.info('[build] Building in parallel mode');
} else {
setImmediate(next);
winston.info('[build] Building in series mode');
}
startTime = Date.now();
buildTargets(targets, parallel, next);
},
function (next) {
async.eachSeries(targets, function (target, next) {
var startTime;
switch (target) {
case 'js':
setImmediate(next);
break;
case 'clientCSS':
winston.info('[build] Building client-side CSS');
startTime = Date.now();
meta.css.minify('client', step.bind(this, startTime, target, next));
break;
case 'acpCSS':
winston.info('[build] Building admin control panel CSS');
startTime = Date.now();
meta.css.minify('admin', step.bind(this, startTime, target, next));
break;
case 'tpl':
winston.info('[build] Building templates');
startTime = Date.now();
meta.templates.compile(step.bind(this, startTime, target, next));
break;
case 'lang':
winston.info('[build] Building language files');
startTime = Date.now();
meta.languages.build(step.bind(this, startTime, target, next));
break;
case 'sound':
winston.info('[build] Linking sound files');
startTime = Date.now();
meta.sounds.build(step.bind(this, startTime, target, next));
break;
default:
winston.warn('[build] Unknown build target: \'' + target + '\'');
setImmediate(next);
break;
}
}, next);
totalTime = (Date.now() - startTime) / 1000;
cacheBuster.write(next);
},
], function (err) {
if (err) {
winston.error('[build] Encountered error during build step: ' + err.message);
return process.exit(1);
return callback(err);
}
cacheBuster.write(function (err) {
if (err) {
winston.error('[build] Failed to write `cache-buster.conf`: ' + err.message);
return process.exit(1);
}
var time = (Date.now() - buildStart) / 1000;
winston.info('[build] Asset compilation successful. Completed in ' + totalTime + 'sec.');
callback();
});
}
winston.info('[build] Asset compilation successful. Completed in ' + time + 's.');
exports.build = build;
if (typeof callback === 'function') {
callback();
} else {
process.exit(0);
}
});
});
exports.buildAll = function (callback) {
build(allTargets, callback);
};

163
src/meta/css.js
Unescape Escape View File

@ -4,15 +4,12 @@ var winston = require('winston');
var nconf = require('nconf');
var fs = require('fs');
var path = require('path');
var less = require('less');
var async = require('async');
var autoprefixer = require('autoprefixer');
var postcss = require('postcss');
var clean = require('postcss-clean');
var plugins = require('../plugins');
var db = require('../database');
var file = require('../file');
var minifier = require('./minifier');
module.exports = function (Meta) {
Meta.css = {};
@ -49,50 +46,19 @@ module.exports = function (Meta) {
},
};
Meta.css.minify = function (target, callback) {
callback = callback || function () {};
winston.verbose('[meta/css] Minifying LESS/CSS');
db.getObjectFields('config', ['theme:type', 'theme:id'], function (err, themeData) {
if (err) {
return callback(err);
}
var themeId = (themeData['theme:id'] || 'nodebb-theme-persona');
var baseThemePath = path.join(nconf.get('themes_path'), (themeData['theme:type'] && themeData['theme:type'] === 'local' ? themeId : 'nodebb-theme-vanilla'));
var paths = [
baseThemePath,
path.join(__dirname, '../../node_modules'),
path.join(__dirname, '../../public/vendor/fontawesome/less'),
];
var source = '';
var lessFiles = filterMissingFiles(plugins.lessFiles);
var cssFiles = filterMissingFiles(plugins.cssFiles);
async.waterfall([
function (next) {
getStyleSource(cssFiles, '\n@import (inline) ".', '.css', next);
},
function (src, next) {
source += src;
getStyleSource(lessFiles, '\n@import ".', '.less', next);
},
function (src, next) {
source += src;
next();
},
], function (err) {
if (err) {
return callback(err);
function filterMissingFiles(filepaths, callback) {
async.filter(filepaths, function (filepath, next) {
file.exists(path.join(__dirname, '../../node_modules', filepath), function (err, exists) {
if (!exists) {
winston.warn('[meta/css] File not found! ' + filepath);
}
minify(buildImports[target](source), paths, target, callback);
next(err, exists);
});
});
};
}, callback);
}
function getStyleSource(files, prefix, extension, callback) {
function getImports(files, prefix, extension, callback) {
var pluginDirectories = [];
var source = '';
@ -121,55 +87,82 @@ module.exports = function (Meta) {
});
}
Meta.css.commitToFile = function (target, source, callback) {
var filename = (target === 'client' ? 'stylesheet' : 'admin') + '.css';
function getBundleMetadata(target, callback) {
var paths = [
path.join(__dirname, '../../node_modules'),
path.join(__dirname, '../../public/vendor/fontawesome/less'),
];
fs.writeFile(path.join(__dirname, '../../build/public/' + filename), source, function (err) {
if (!err) {
winston.verbose('[meta/css] ' + target + ' CSS committed to disk.');
} else {
winston.error('[meta/css] ' + err.message);
process.exit(1);
}
async.waterfall([
function (next) {
if (target !== 'client') {
return next(null, null);
}
callback();
});
};
db.getObjectFields('config', ['theme:type', 'theme:id'], next);
},
function (themeData, next) {
if (target === 'client') {
var themeId = (themeData['theme:id'] || 'nodebb-theme-persona');
var baseThemePath = path.join(nconf.get('themes_path'), (themeData['theme:type'] && themeData['theme:type'] === 'local' ? themeId : 'nodebb-theme-vanilla'));
paths.unshift(baseThemePath);
}
function minify(source, paths, target, callback) {
callback = callback || function () {};
less.render(source, {
paths: paths,
}, function (err, lessOutput) {
async.parallel({
less: function (cb) {
async.waterfall([
function (next) {
filterMissingFiles(plugins.lessFiles, next);
},
function (lessFiles, next) {
getImports(lessFiles, '\n@import ".', '.less', next);
},
], cb);
},
css: function (cb) {
async.waterfall([
function (next) {
filterMissingFiles(plugins.cssFiles, next);
},
function (cssFiles, next) {
getImports(cssFiles, '\n@import (inline) ".', '.css', next);
},
], cb);
},
}, next);
},
function (result, next) {
var cssImports = result.css;
var lessImports = result.less;
var imports = cssImports + '\n' + lessImports;
imports = buildImports[target](imports);
next(null, imports);
},
], function (err, imports) {
if (err) {
winston.error('[meta/css] Could not minify LESS/CSS: ' + err.message);
return callback(err);
}
postcss(global.env === 'development' ? [autoprefixer] : [
autoprefixer,
clean({
processImportFrom: ['local'],
}),
]).process(lessOutput.css).then(function (result) {
result.warnings().forEach(function (warn) {
winston.verbose(warn.toString());
});
return Meta.css.commitToFile(target, result.css, function () {
callback(null, result.css);
});
});
callback(null, { paths: paths, imports: imports });
});
}
function filterMissingFiles(files) {
return files.filter(function (filePath) {
var exists = file.existsSync(path.join(__dirname, '../../node_modules', filePath));
if (!exists) {
winston.warn('[meta/css] File not found! ' + filePath);
}
return exists;
});
}
Meta.css.buildBundle = function (target, fork, callback) {
async.waterfall([
function (next) {
getBundleMetadata(target, next);
},
function (data, next) {
var minify = global.env !== 'development';
minifier.css.bundle(data.imports, data.paths, minify, fork, next);
},
function (bundle, next) {
var filename = (target === 'client' ? 'stylesheet' : 'admin') + '.css';
fs.writeFile(path.join(__dirname, '../../build/public', filename), bundle.code, next);
},
], callback);
};
};

314
src/meta/js.js
Unescape Escape View File

@ -1,136 +1,128 @@
'use strict';
var winston = require('winston');
var fork = require('child_process').fork;
var path = require('path');
var async = require('async');
var fs = require('fs');
var mkdirp = require('mkdirp');
var rimraf = require('rimraf');
var uglifyjs = require('uglify-js');
var file = require('../file');
var plugins = require('../plugins');
var minifierPath = path.join(__dirname, 'minifier.js');
var minifier = require('./minifier');
module.exports = function (Meta) {
Meta.js = {
target: {},
scripts: {
base: [
'node_modules/jquery/dist/jquery.js',
'node_modules/socket.io-client/dist/socket.io.js',
'public/vendor/jquery/timeago/jquery.timeago.js',
'public/vendor/jquery/js/jquery.form.min.js',
'public/vendor/visibility/visibility.min.js',
'node_modules/bootstrap/dist/js/bootstrap.js',
'public/vendor/jquery/bootstrap-tagsinput/bootstrap-tagsinput.min.js',
'public/vendor/jquery/textcomplete/jquery.textcomplete.js',
'public/vendor/requirejs/require.js',
'public/src/require-config.js',
'public/vendor/bootbox/bootbox.js',
'public/vendor/bootbox/wrapper.js',
'public/vendor/tinycon/tinycon.js',
'public/vendor/xregexp/xregexp.js',
'public/vendor/xregexp/unicode/unicode-base.js',
'node_modules/templates.js/lib/templates.js',
'public/src/utils.js',
'public/src/sockets.js',
'public/src/app.js',
'public/src/ajaxify.js',
'public/src/overrides.js',
'public/src/widgets.js',
'node_modules/promise-polyfill/promise.js',
],
// files listed below are only available client-side, or are bundled in to reduce # of network requests on cold load
rjs: [
'public/src/client/footer.js',
'public/src/client/chats.js',
'public/src/client/infinitescroll.js',
'public/src/client/pagination.js',
'public/src/client/recent.js',
'public/src/client/unread.js',
'public/src/client/topic.js',
'public/src/client/topic/events.js',
'public/src/client/topic/fork.js',
'public/src/client/topic/move.js',
'public/src/client/topic/posts.js',
'public/src/client/topic/images.js',
'public/src/client/topic/postTools.js',
'public/src/client/topic/threadTools.js',
'public/src/client/categories.js',
'public/src/client/category.js',
'public/src/client/category/tools.js',
'public/src/modules/translator.js',
'public/src/modules/notifications.js',
'public/src/modules/chat.js',
'public/src/modules/components.js',
'public/src/modules/sort.js',
'public/src/modules/navigator.js',
'public/src/modules/topicSelect.js',
'public/src/modules/share.js',
'public/src/modules/search.js',
'public/src/modules/alerts.js',
'public/src/modules/taskbar.js',
'public/src/modules/helpers.js',
'public/src/modules/string.js',
'public/src/modules/flags.js',
'public/src/modules/storage.js',
],
// modules listed below are built (/src/modules) so they can be defined anonymously
modules: {
'Chart.js': 'node_modules/chart.js/dist/Chart.min.js',
'mousetrap.js': 'node_modules/mousetrap/mousetrap.min.js',
'cropper.js': 'node_modules/cropperjs/dist/cropper.min.js',
'jqueryui.js': 'public/vendor/jquery/js/jquery-ui.js',
'zxcvbn.js': 'node_modules/zxcvbn/dist/zxcvbn.js',
ace: 'node_modules/ace-builds/src-min',
},
Meta.js = {};
Meta.js.scripts = {
base: [
'node_modules/jquery/dist/jquery.js',
'node_modules/socket.io-client/dist/socket.io.js',
'public/vendor/jquery/timeago/jquery.timeago.js',
'public/vendor/jquery/js/jquery.form.min.js',
'public/vendor/visibility/visibility.min.js',
'node_modules/bootstrap/dist/js/bootstrap.js',
'public/vendor/jquery/bootstrap-tagsinput/bootstrap-tagsinput.min.js',
'public/vendor/jquery/textcomplete/jquery.textcomplete.js',
'public/vendor/requirejs/require.js',
'public/src/require-config.js',
'public/vendor/bootbox/bootbox.js',
'public/vendor/bootbox/wrapper.js',
'public/vendor/tinycon/tinycon.js',
'public/vendor/xregexp/xregexp.js',
'public/vendor/xregexp/unicode/unicode-base.js',
'node_modules/templates.js/lib/templates.js',
'public/src/utils.js',
'public/src/sockets.js',
'public/src/app.js',
'public/src/ajaxify.js',
'public/src/overrides.js',
'public/src/widgets.js',
'node_modules/promise-polyfill/promise.js',
],
// files listed below are only available client-side, or are bundled in to reduce # of network requests on cold load
rjs: [
'public/src/client/footer.js',
'public/src/client/chats.js',
'public/src/client/infinitescroll.js',
'public/src/client/pagination.js',
'public/src/client/recent.js',
'public/src/client/unread.js',
'public/src/client/topic.js',
'public/src/client/topic/events.js',
'public/src/client/topic/fork.js',
'public/src/client/topic/move.js',
'public/src/client/topic/posts.js',
'public/src/client/topic/images.js',
'public/src/client/topic/postTools.js',
'public/src/client/topic/threadTools.js',
'public/src/client/categories.js',
'public/src/client/category.js',
'public/src/client/category/tools.js',
'public/src/modules/translator.js',
'public/src/modules/notifications.js',
'public/src/modules/chat.js',
'public/src/modules/components.js',
'public/src/modules/sort.js',
'public/src/modules/navigator.js',
'public/src/modules/topicSelect.js',
'public/src/modules/share.js',
'public/src/modules/search.js',
'public/src/modules/alerts.js',
'public/src/modules/taskbar.js',
'public/src/modules/helpers.js',
'public/src/modules/string.js',
'public/src/modules/flags.js',
'public/src/modules/storage.js',
],
// modules listed below are built (/src/modules) so they can be defined anonymously
modules: {
'Chart.js': 'node_modules/chart.js/dist/Chart.min.js',
'mousetrap.js': 'node_modules/mousetrap/mousetrap.min.js',
'cropper.js': 'node_modules/cropperjs/dist/cropper.min.js',
'jqueryui.js': 'public/vendor/jquery/js/jquery-ui.js',
'zxcvbn.js': 'node_modules/zxcvbn/dist/zxcvbn.js',
ace: 'node_modules/ace-builds/src-min',
},
};
function minifyModules(modules, callback) {
function minifyModules(modules, fork, callback) {
// for it to never fork
// otherwise it spawns way too many processes
// maybe eventually we can pool modules
// and pass the pools to the minifer
// to reduce the total number of threads
fork = false;
async.eachLimit(modules, 500, function (mod, next) {
var srcPath = mod.srcPath;
var destPath = mod.destPath;
var minified;
async.parallel([
function (cb) {
async.parallel({
dirped: function (cb) {
mkdirp(path.dirname(destPath), cb);
},
function (cb) {
minified: function (cb) {
fs.readFile(srcPath, function (err, buffer) {
if (err) {
return cb(err);
}
if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) {
minified = { code: buffer.toString() };
return cb();
}
try {
minified = uglifyjs.minify(buffer.toString(), {
fromString: true,
compress: false,
});
} catch (e) {
return cb(e);
return cb(null, { code: buffer.toString() });
}
cb();
minifier.js.minify(buffer.toString(), fork, cb);
});
},
], function (err) {
}, function (err, results) {
if (err) {
return next(err);
}
var minified = results.minified;
fs.writeFile(destPath, minified.code, next);
});
}, callback);
@ -233,7 +225,7 @@ module.exports = function (Meta) {
});
}
Meta.js.buildModules = function (callback) {
Meta.js.buildModules = function (fork, callback) {
async.waterfall([
clearModules,
function (next) {
@ -244,7 +236,7 @@ module.exports = function (Meta) {
getModuleList(next);
},
function (modules, next) {
minifyModules(modules, next);
minifyModules(modules, fork, next);
},
], callback);
};
@ -269,52 +261,13 @@ module.exports = function (Meta) {
});
};
Meta.js.minify = function (target, callback) {
winston.verbose('[meta/js] Minifying ' + target);
var forkProcessParams = setupDebugging();
var minifier = fork(minifierPath, [], forkProcessParams);
Meta.js.minifierProc = minifier;
Meta.js.target[target] = {};
Meta.js.prepare(target, function (err) {
if (err) {
return callback(err);
}
minifier.send({
action: 'js',
minify: global.env !== 'development',
scripts: Meta.js.target[target].scripts,
});
});
minifier.on('message', function (message) {
switch (message.type) {
case 'end':
Meta.js.target[target].cache = message.minified;
Meta.js.target[target].map = message.sourceMap;
winston.verbose('[meta/js] ' + target + ' minification complete');
minifier.kill();
Meta.js.commitToFile(target, callback);
break;
case 'error':
winston.error('[meta/js] Could not compile ' + target + ': ' + message.message);
minifier.kill();
callback(new Error(message.message));
break;
}
});
};
Meta.js.prepare = function (target, callback) {
var pluginsScripts = [];
function getBundleScriptList(target, callback) {
var pluginDirectories = [];
pluginsScripts = plugins[target === 'nodebb.min.js' ? 'clientScripts' : 'acpScripts'].filter(function (path) {
if (target === 'admin') {
target = 'acp';
}
var pluginScripts = plugins[target + 'Scripts'].filter(function (path) {
if (path.endsWith('.js')) {
return true;
}
@ -325,8 +278,12 @@ module.exports = function (Meta) {
async.each(pluginDirectories, function (directory, next) {
file.walk(directory, function (err, scripts) {
pluginsScripts = pluginsScripts.concat(scripts);
next(err);
if (err) {
return next(err);
}
pluginScripts = pluginScripts.concat(scripts);
next();
});
}, function (err) {
if (err) {
@ -335,52 +292,43 @@ module.exports = function (Meta) {
var basePath = path.resolve(__dirname, '../..');
Meta.js.target[target].scripts = Meta.js.scripts.base.concat(pluginsScripts);
var scripts = Meta.js.scripts.base.concat(pluginScripts);
if (target === 'nodebb.min.js') {
Meta.js.target[target].scripts = Meta.js.target[target].scripts.concat(Meta.js.scripts.rjs);
if (target === 'client' && global.env !== 'development') {
scripts = scripts.concat(Meta.js.scripts.rjs);
}
Meta.js.target[target].scripts = Meta.js.target[target].scripts.map(function (script) {
scripts = scripts.map(function (script) {
return path.resolve(basePath, script).replace(/\\/g, '/');
});
callback();
callback(null, scripts);
});
};
}
Meta.js.killMinifier = function () {
if (Meta.js.minifierProc) {
Meta.js.minifierProc.kill('SIGTERM');
}
};
Meta.js.buildBundle = function (target, fork, callback) {
var fileNames = {
client: 'nodebb.min.js',
admin: 'acp.min.js',
};
Meta.js.commitToFile = function (target, callback) {
fs.writeFile(path.join(__dirname, '../../build/public', target), Meta.js.target[target].cache, function (err) {
callback(err);
});
};
async.waterfall([
function (next) {
getBundleScriptList(target, next);
},
function (files, next) {
var minify = global.env !== 'development';
function setupDebugging() {
/**
* Check if the parent process is running with the debug option --debug (or --debug-brk)
*/
var forkProcessParams = {};
if (global.v8debug || parseInt(process.execArgv.indexOf('--debug'), 10) !== -1) {
/**
* use the line below if you want to debug minifier.js script too (or even --debug-brk option, but
* you'll have to setup your debugger and connect to the forked process)
*/
// forkProcessParams = {execArgv: ['--debug=' + (global.process.debugPort + 1), '--nolazy']};
/**
* otherwise, just clean up --debug/--debug-brk options which are set up by default from the parent one
*/
forkProcessParams = {
execArgv: [],
};
}
minifier.js.bundle(files, minify, fork, next);
},
function (bundle, next) {
var filePath = path.join(__dirname, '../../build/public', fileNames[target]);
fs.writeFile(filePath, bundle.code, next);
},
], callback);
};
return forkProcessParams;
}
Meta.js.killMinifier = function () {
minifier.killAll();
};
};

285
src/meta/minifier.js
Unescape Escape View File

@ -3,86 +3,255 @@
var uglifyjs = require('uglify-js');
var async = require('async');
var fs = require('fs');
var childProcess = require('child_process');
var os = require('os');
var less = require('less');
var postcss = require('postcss');
var autoprefixer = require('autoprefixer');
var clean = require('postcss-clean');
var file = require('../file');
var Minifier = {
js: {},
var Minifier = module.exports;
function setupDebugging() {
/**
* Check if the parent process is running with the debug option --debug (or --debug-brk)
*/
var forkProcessParams = {};
if (global.v8debug || parseInt(process.execArgv.indexOf('--debug'), 10) !== -1) {
/**
* use the line below if you want to debug minifier.js script too (or even --debug-brk option, but
* you'll have to setup your debugger and connect to the forked process)
*/
// forkProcessParams = { execArgv: ['--debug=' + (global.process.debugPort + 1), '--nolazy'] };
/**
* otherwise, just clean up --debug/--debug-brk options which are set up by default from the parent one
*/
forkProcessParams = {
execArgv: [],
};
}
return forkProcessParams;
}
var children = [];
Minifier.killAll = function () {
children.forEach(function (child) {
child.kill('SIGTERM');
});
children = [];
};
/* Javascript */
Minifier.js.minify = function (scripts, minify, callback) {
scripts = scripts.filter(function (file) {
return file && file.endsWith('.js');
function removeChild(proc) {
children = children.filter(function (child) {
return child !== proc;
});
}
async.filter(scripts, function (script, next) {
file.exists(script, function (err, exists) {
if (err) {
return next(err);
}
function forkAction(action, callback) {
var forkProcessParams = setupDebugging();
var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, {
cwd: __dirname,
env: {
minifier_child: true,
},
}));
if (!exists) {
console.warn('[minifier] file not found, ' + script);
}
next(null, exists);
});
}, function (err, scripts) {
if (err) {
return callback(err);
children.push(proc);
proc.on('message', function (message) {
if (message.type === 'error') {
proc.kill();
return callback(new Error(message.message));
}
if (minify) {
minifyScripts(scripts, callback);
} else {
concatenateScripts(scripts, callback);
if (message.type === 'end') {
proc.kill();
callback(null, message.result);
}
});
};
proc.on('error', function (err) {
proc.kill();
removeChild(proc);
callback(err);
});
process.on('message', function (payload) {
switch (payload.action) {
case 'js':
Minifier.js.minify(payload.scripts, payload.minify, function (minified/* , sourceMap*/) {
process.send({
type: 'end',
// sourceMap: sourceMap,
minified: minified,
proc.send({
type: 'action',
action: action,
});
proc.on('close', function () {
removeChild(proc);
});
}
var actions = {};
if (process.env.minifier_child) {
process.on('message', function (message) {
if (message.type === 'action') {
var action = message.action;
if (typeof actions[action.act] !== 'function') {
process.send({
type: 'error',
message: 'Unknown action',
});
return;
}
actions[action.act](action, function (err, result) {
if (err) {
process.send({
type: 'error',
message: err.message,
});
return;
}
process.send({
type: 'end',
result: result,
});
});
});
break;
}
});
}
function executeAction(action, fork, callback) {
if (fork) {
forkAction(action, callback);
} else {
if (typeof actions[action.act] !== 'function') {
return callback(Error('Unknown action'));
}
actions[action.act](action, callback);
}
});
function minifyScripts(scripts, callback) {
// The portions of code involving the source map are commented out as they're broken in UglifyJS2
// Follow along here: https://github.com/mishoo/UglifyJS2/issues/700
try {
var minified = uglifyjs.minify(scripts, {
// outSourceMap: "nodebb.min.js.map",
compress: false,
});
}
function concat(data, callback) {
if (data.files && data.files.length) {
async.mapLimit(data.files, 1000, fs.readFile, function (err, files) {
if (err) {
return callback(err);
}
callback(minified.code/* , minified.map*/);
} catch (err) {
process.send({
type: 'error',
message: err.message,
var output = files.join(os.EOL + ';');
callback(null, { code: output });
});
return;
}
callback();
}
actions.concat = concat;
function concatenateScripts(scripts, callback) {
async.map(scripts, fs.readFile, function (err, scripts) {
if (err) {
process.send({
type: 'error',
message: err.message,
function minifyJS(data, callback) {
var minified;
if (data.fromSource) {
var sources = data.source;
var multiple = Array.isArray(sources);
if (!multiple) {
sources = [sources];
}
try {
minified = sources.map(function (source) {
return uglifyjs.minify(source, {
// outSourceMap: data.filename + '.map',
compress: data.compress,
fromString: true,
output: {
// suppress uglify line length warnings
max_line_len: 400000,
},
});
});
return;
} catch (e) {
return callback(e);
}
scripts = scripts.join(require('os').EOL + ';');
return callback(null, multiple ? minified : minified[0]);
}
if (data.files && data.files.length) {
async.filter(data.files, file.exists, function (err, scripts) {
if (err) {
return callback(err);
}
try {
minified = uglifyjs.minify(scripts, {
// outSourceMap: data.filename + '.map',
compress: data.compress,
fromString: false,
});
} catch (e) {
return callback(e);
}
callback(null, minified);
});
callback(scripts);
return;
}
callback();
}
actions.minifyJS = minifyJS;
Minifier.js = {};
Minifier.js.bundle = function (scripts, minify, fork, callback) {
executeAction({
act: minify ? 'minifyJS' : 'concat',
files: scripts,
compress: false,
}, fork, callback);
};
Minifier.js.minify = function (source, fork, callback) {
executeAction({
act: 'minifyJS',
fromSource: true,
source: source,
}, fork, callback);
};
function buildCSS(data, callback) {
less.render(data.source, {
paths: data.paths,
}, function (err, lessOutput) {
if (err) {
return callback(err);
}
postcss(data.minify ? [
autoprefixer,
clean({
processImportFrom: ['local'],
}),
] : [autoprefixer]).process(lessOutput.css).then(function (result) {
callback(null, { code: result.css });
}, function (err) {
callback(err);
});
});
}
actions.buildCSS = buildCSS;
Minifier.css = {};
Minifier.css.bundle = function (source, paths, minify, fork, callback) {
executeAction({
act: 'buildCSS',
source: source,
paths: paths,
minify: minify,
}, fork, callback);
};

Write Preview
Loading…
Cancel
Save