diff --git a/package.json b/package.json
index ec678b10a5..4be80a1ce1 100644
--- a/package.json
+++ b/package.json
@@ -17,8 +17,8 @@
"coveralls": "istanbul cover _mocha --report lcovonly -- -R dot && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage"
},
"dependencies": {
- "async": "2.4.0",
"ace-builds": "^1.2.6",
+ "async": "2.4.0",
"autoprefixer": "7.0.1",
"bcryptjs": "2.4.3",
"body-parser": "^1.9.0",
@@ -45,6 +45,7 @@
"jquery": "^3.1.0",
"json-2-csv": "^2.0.22",
"less": "^2.0.0",
+ "lodash.padstart": "^4.6.1",
"logrotate-stream": "^0.2.3",
"lru-cache": "4.0.2",
"mime": "^1.3.4",
diff --git a/public/less/admin/admin.less b/public/less/admin/admin.less
index a37682ab83..8f713028ed 100644
--- a/public/less/admin/admin.less
+++ b/public/less/admin/admin.less
@@ -1,4 +1,4 @@
-@import "../../node_modules/bootstrap/less/bootstrap";
+@import "bootstrap/less/bootstrap";
@import "./paper/variables";
@import "./paper/bootswatch";
@import "./mixins";
diff --git a/src/meta/build.js b/src/meta/build.js
index 9ba5ec89bf..e88bbb17e9 100644
--- a/src/meta/build.js
+++ b/src/meta/build.js
@@ -2,149 +2,212 @@
var async = require('async');
var winston = require('winston');
+var os = require('os');
+var nconf = require('nconf');
+var padstart = require('lodash.padstart');
-var buildStart;
+var cacheBuster = require('./cacheBuster');
+var meta;
-var valid = ['js', 'clientCSS', 'acpCSS', 'tpl', 'lang', 'sound'];
+function step(target, callback) {
+ var startTime = Date.now();
+ winston.info('[build] ' + target + ' build started');
-exports.buildAll = function (callback) {
- exports.build(valid.join(','), callback);
-};
+ return function (err) {
+ if (err) {
+ winston.error('[build] ' + target + ' build failed');
+ return callback(err);
+ }
-exports.build = function build(targets, callback) {
- buildStart = Date.now();
+ var time = (Date.now() - startTime) / 1000;
- var db = require('../database');
- var meta = require('../meta');
- var plugins = require('../plugins');
+ winston.info('[build] ' + target + ' build completed in ' + time + 'sec');
+ callback();
+ };
+}
+
+var targetHandlers = {
+ 'plugin static dirs': function (parallel, callback) {
+ meta.js.linkStatics(callback);
+ },
+ 'requirejs modules': function (parallel, callback) {
+ meta.js.buildModules(parallel, callback);
+ },
+ 'client js bundle': function (parallel, callback) {
+ meta.js.buildBundle('client', parallel, callback);
+ },
+ 'admin js bundle': function (parallel, callback) {
+ meta.js.buildBundle('admin', parallel, callback);
+ },
+ javascript: [
+ 'plugin static dirs',
+ 'requirejs modules',
+ 'client js bundle',
+ 'admin js bundle',
+ ],
+ 'client side styles': function (parallel, callback) {
+ meta.css.buildBundle('client', parallel, callback);
+ },
+ 'admin control panel styles': function (parallel, callback) {
+ meta.css.buildBundle('admin', parallel, callback);
+ },
+ styles: [
+ 'client side styles',
+ 'admin control panel styles',
+ ],
+ templates: function (parallel, callback) {
+ meta.templates.compile(callback);
+ },
+ languages: function (parallel, callback) {
+ meta.languages.build(callback);
+ },
+ sounds: function (parallel, callback) {
+ meta.sounds.build(callback);
+ },
+};
+var aliases = {
+ 'plugin static dirs': ['staticdirs'],
+ 'requirejs modules': ['rjs', 'modules'],
+ 'client js bundle': ['clientjs', 'clientscript', 'clientscripts'],
+ 'admin js bundle': ['adminjs', 'adminscript', 'adminscripts'],
+ javascript: ['js'],
+ 'client side styles': [
+ 'clientcss', 'clientless', 'clientstyles', 'clientstyle',
+ ],
+ 'admin control panel styles': [
+ 'admincss', 'adminless', 'adminstyles', 'adminstyle', 'acpcss', 'acpless', 'acpstyles', 'acpstyle',
+ ],
+ styles: ['css', 'less', 'style'],
+ templates: ['tpl'],
+ languages: ['lang', 'i18n'],
+ sounds: ['sound'],
+};
- targets = (targets === true ? valid : targets.split(',').filter(function (target) {
- return valid.indexOf(target) !== -1;
- }));
+aliases = Object.keys(aliases).reduce(function (prev, key) {
+ var arr = aliases[key];
+ arr.forEach(function (alias) {
+ prev[alias] = key;
+ });
+ prev[key] = key;
+ return prev;
+}, {});
- if (!targets) {
- winston.error('[build] No valid build targets found. Aborting.');
- return process.exit(0);
- }
+function beforeBuild(callback) {
+ var db = require('../database');
+ var plugins = require('../plugins');
+ meta = require('../meta');
async.series([
- async.apply(db.init),
- async.apply(meta.themes.setupPaths),
- async.apply(plugins.prepareForBuild),
+ db.init,
+ meta.themes.setupPaths,
+ plugins.prepareForBuild,
], function (err) {
if (err) {
winston.error('[build] Encountered error preparing for build: ' + err.message);
- return process.exit(1);
+ return callback(err);
}
- exports.buildTargets(targets, callback);
+ callback();
});
-};
+}
-exports.buildTargets = function (targets, callback) {
- var cacheBuster = require('./cacheBuster');
- var meta = require('../meta');
- var numCpus = require('os').cpus().length;
- var parallel = targets.length > 1 && numCpus > 1;
+var allTargets = Object.keys(targetHandlers).filter(function (name) {
+ return typeof targetHandlers[name] === 'function';
+});
+function buildTargets(targets, parallel, callback) {
+ var all = parallel ? async.each : async.eachSeries;
- buildStart = buildStart || Date.now();
+ var length = Math.max.apply(Math, targets.map(function (name) {
+ return name.length;
+ }));
- var step = function (startTime, target, next, err) {
- if (err) {
- winston.error('Build failed: ' + err.stack);
- process.exit(1);
- }
- winston.info('[build] ' + target + ' => Completed in ' + ((Date.now() - startTime) / 1000) + 's');
- next();
- };
+ all(targets, function (target, next) {
+ targetHandlers[target](parallel, step(padstart(target, length) + ' ', next));
+ }, callback);
+}
- if (parallel) {
- winston.verbose('[build] Utilising multiple cores/processes');
- } else {
- winston.verbose('[build] Utilising single-core');
+function build(targets, callback) {
+ if (targets === true) {
+ targets = allTargets;
+ } else if (!Array.isArray(targets)) {
+ targets = targets.split(',');
}
- async[parallel ? 'parallel' : 'series']([
+ targets = targets
+ // get full target name
+ .map(function (target) {
+ target = target.toLowerCase().replace(/-/g, '');
+ if (!aliases[target]) {
+ winston.warn('[build] Unknown target: ' + target);
+ return false;
+ }
+
+ return aliases[target];
+ })
+ // filter nonexistent targets
+ .filter(Boolean)
+ // map multitargets to their sets
+ .reduce(function (prev, target) {
+ if (Array.isArray(targetHandlers[target])) {
+ return prev.concat(targetHandlers[target]);
+ }
+
+ return prev.concat(target);
+ }, [])
+ // unique
+ .filter(function (target, i, arr) {
+ return arr.indexOf(target) === i;
+ });
+
+ if (typeof callback !== 'function') {
+ callback = function (err) {
+ if (err) {
+ winston.error(err);
+ process.exit(1);
+ } else {
+ process.exit(0);
+ }
+ };
+ }
+
+ if (!targets) {
+ winston.info('[build] No valid targets supplied. Aborting.');
+ callback();
+ }
+
+ var startTime;
+ var totalTime;
+ async.series([
+ beforeBuild,
function (next) {
- if (targets.indexOf('js') !== -1) {
- winston.info('[build] Building javascript');
- var startTime = Date.now();
- async.series([
- meta.js.buildModules,
- meta.js.linkStatics,
- async.apply(meta.js.minify, 'nodebb.min.js'),
- async.apply(meta.js.minify, 'acp.min.js'),
- ], step.bind(this, startTime, 'js', next));
+ var parallel = os.cpus().length > 1 && !nconf.get('series');
+ if (parallel) {
+ winston.info('[build] Building in parallel mode');
} else {
- setImmediate(next);
+ winston.info('[build] Building in series mode');
}
+
+ startTime = Date.now();
+ buildTargets(targets, parallel, next);
},
function (next) {
- async.eachSeries(targets, function (target, next) {
- var startTime;
- switch (target) {
- case 'js':
- setImmediate(next);
- break;
- case 'clientCSS':
- winston.info('[build] Building client-side CSS');
- startTime = Date.now();
- meta.css.minify('client', step.bind(this, startTime, target, next));
- break;
-
- case 'acpCSS':
- winston.info('[build] Building admin control panel CSS');
- startTime = Date.now();
- meta.css.minify('admin', step.bind(this, startTime, target, next));
- break;
-
- case 'tpl':
- winston.info('[build] Building templates');
- startTime = Date.now();
- meta.templates.compile(step.bind(this, startTime, target, next));
- break;
-
- case 'lang':
- winston.info('[build] Building language files');
- startTime = Date.now();
- meta.languages.build(step.bind(this, startTime, target, next));
- break;
-
- case 'sound':
- winston.info('[build] Linking sound files');
- startTime = Date.now();
- meta.sounds.build(step.bind(this, startTime, target, next));
- break;
-
- default:
- winston.warn('[build] Unknown build target: \'' + target + '\'');
- setImmediate(next);
- break;
- }
- }, next);
+ totalTime = (Date.now() - startTime) / 1000;
+ cacheBuster.write(next);
},
], function (err) {
if (err) {
winston.error('[build] Encountered error during build step: ' + err.message);
- return process.exit(1);
+ return callback(err);
}
- cacheBuster.write(function (err) {
- if (err) {
- winston.error('[build] Failed to write `cache-buster.conf`: ' + err.message);
- return process.exit(1);
- }
-
- var time = (Date.now() - buildStart) / 1000;
+ winston.info('[build] Asset compilation successful. Completed in ' + totalTime + 'sec.');
+ callback();
+ });
+}
- winston.info('[build] Asset compilation successful. Completed in ' + time + 's.');
+exports.build = build;
- if (typeof callback === 'function') {
- callback();
- } else {
- process.exit(0);
- }
- });
- });
+exports.buildAll = function (callback) {
+ build(allTargets, callback);
};
diff --git a/src/meta/css.js b/src/meta/css.js
index 88ad3348e8..127191437c 100644
--- a/src/meta/css.js
+++ b/src/meta/css.js
@@ -4,15 +4,12 @@ var winston = require('winston');
var nconf = require('nconf');
var fs = require('fs');
var path = require('path');
-var less = require('less');
var async = require('async');
-var autoprefixer = require('autoprefixer');
-var postcss = require('postcss');
-var clean = require('postcss-clean');
var plugins = require('../plugins');
var db = require('../database');
var file = require('../file');
+var minifier = require('./minifier');
module.exports = function (Meta) {
Meta.css = {};
@@ -49,50 +46,19 @@ module.exports = function (Meta) {
},
};
- Meta.css.minify = function (target, callback) {
- callback = callback || function () {};
-
- winston.verbose('[meta/css] Minifying LESS/CSS');
- db.getObjectFields('config', ['theme:type', 'theme:id'], function (err, themeData) {
- if (err) {
- return callback(err);
- }
-
- var themeId = (themeData['theme:id'] || 'nodebb-theme-persona');
- var baseThemePath = path.join(nconf.get('themes_path'), (themeData['theme:type'] && themeData['theme:type'] === 'local' ? themeId : 'nodebb-theme-vanilla'));
- var paths = [
- baseThemePath,
- path.join(__dirname, '../../node_modules'),
- path.join(__dirname, '../../public/vendor/fontawesome/less'),
- ];
- var source = '';
-
- var lessFiles = filterMissingFiles(plugins.lessFiles);
- var cssFiles = filterMissingFiles(plugins.cssFiles);
-
- async.waterfall([
- function (next) {
- getStyleSource(cssFiles, '\n@import (inline) ".', '.css', next);
- },
- function (src, next) {
- source += src;
- getStyleSource(lessFiles, '\n@import ".', '.less', next);
- },
- function (src, next) {
- source += src;
- next();
- },
- ], function (err) {
- if (err) {
- return callback(err);
+ function filterMissingFiles(filepaths, callback) {
+ async.filter(filepaths, function (filepath, next) {
+ file.exists(path.join(__dirname, '../../node_modules', filepath), function (err, exists) {
+ if (!exists) {
+ winston.warn('[meta/css] File not found! ' + filepath);
}
- minify(buildImports[target](source), paths, target, callback);
+ next(err, exists);
});
- });
- };
+ }, callback);
+ }
- function getStyleSource(files, prefix, extension, callback) {
+ function getImports(files, prefix, extension, callback) {
var pluginDirectories = [];
var source = '';
@@ -121,55 +87,82 @@ module.exports = function (Meta) {
});
}
- Meta.css.commitToFile = function (target, source, callback) {
- var filename = (target === 'client' ? 'stylesheet' : 'admin') + '.css';
+ function getBundleMetadata(target, callback) {
+ var paths = [
+ path.join(__dirname, '../../node_modules'),
+ path.join(__dirname, '../../public/vendor/fontawesome/less'),
+ ];
- fs.writeFile(path.join(__dirname, '../../build/public/' + filename), source, function (err) {
- if (!err) {
- winston.verbose('[meta/css] ' + target + ' CSS committed to disk.');
- } else {
- winston.error('[meta/css] ' + err.message);
- process.exit(1);
- }
+ async.waterfall([
+ function (next) {
+ if (target !== 'client') {
+ return next(null, null);
+ }
- callback();
- });
- };
+ db.getObjectFields('config', ['theme:type', 'theme:id'], next);
+ },
+ function (themeData, next) {
+ if (target === 'client') {
+ var themeId = (themeData['theme:id'] || 'nodebb-theme-persona');
+ var baseThemePath = path.join(nconf.get('themes_path'), (themeData['theme:type'] && themeData['theme:type'] === 'local' ? themeId : 'nodebb-theme-vanilla'));
+ paths.unshift(baseThemePath);
+ }
- function minify(source, paths, target, callback) {
- callback = callback || function () {};
- less.render(source, {
- paths: paths,
- }, function (err, lessOutput) {
+ async.parallel({
+ less: function (cb) {
+ async.waterfall([
+ function (next) {
+ filterMissingFiles(plugins.lessFiles, next);
+ },
+ function (lessFiles, next) {
+ getImports(lessFiles, '\n@import ".', '.less', next);
+ },
+ ], cb);
+ },
+ css: function (cb) {
+ async.waterfall([
+ function (next) {
+ filterMissingFiles(plugins.cssFiles, next);
+ },
+ function (cssFiles, next) {
+ getImports(cssFiles, '\n@import (inline) ".', '.css', next);
+ },
+ ], cb);
+ },
+ }, next);
+ },
+ function (result, next) {
+ var cssImports = result.css;
+ var lessImports = result.less;
+
+ var imports = cssImports + '\n' + lessImports;
+ imports = buildImports[target](imports);
+
+ next(null, imports);
+ },
+ ], function (err, imports) {
if (err) {
- winston.error('[meta/css] Could not minify LESS/CSS: ' + err.message);
return callback(err);
}
- postcss(global.env === 'development' ? [autoprefixer] : [
- autoprefixer,
- clean({
- processImportFrom: ['local'],
- }),
- ]).process(lessOutput.css).then(function (result) {
- result.warnings().forEach(function (warn) {
- winston.verbose(warn.toString());
- });
-
- return Meta.css.commitToFile(target, result.css, function () {
- callback(null, result.css);
- });
- });
+ callback(null, { paths: paths, imports: imports });
});
}
- function filterMissingFiles(files) {
- return files.filter(function (filePath) {
- var exists = file.existsSync(path.join(__dirname, '../../node_modules', filePath));
- if (!exists) {
- winston.warn('[meta/css] File not found! ' + filePath);
- }
- return exists;
- });
- }
+ Meta.css.buildBundle = function (target, fork, callback) {
+ async.waterfall([
+ function (next) {
+ getBundleMetadata(target, next);
+ },
+ function (data, next) {
+ var minify = global.env !== 'development';
+ minifier.css.bundle(data.imports, data.paths, minify, fork, next);
+ },
+ function (bundle, next) {
+ var filename = (target === 'client' ? 'stylesheet' : 'admin') + '.css';
+
+ fs.writeFile(path.join(__dirname, '../../build/public', filename), bundle.code, next);
+ },
+ ], callback);
+ };
};
diff --git a/src/meta/js.js b/src/meta/js.js
index f654d45644..e7b22939fc 100644
--- a/src/meta/js.js
+++ b/src/meta/js.js
@@ -1,136 +1,128 @@
'use strict';
-var winston = require('winston');
-var fork = require('child_process').fork;
var path = require('path');
var async = require('async');
var fs = require('fs');
var mkdirp = require('mkdirp');
var rimraf = require('rimraf');
-var uglifyjs = require('uglify-js');
var file = require('../file');
var plugins = require('../plugins');
-
-var minifierPath = path.join(__dirname, 'minifier.js');
+var minifier = require('./minifier');
module.exports = function (Meta) {
- Meta.js = {
- target: {},
- scripts: {
- base: [
- 'node_modules/jquery/dist/jquery.js',
- 'node_modules/socket.io-client/dist/socket.io.js',
- 'public/vendor/jquery/timeago/jquery.timeago.js',
- 'public/vendor/jquery/js/jquery.form.min.js',
- 'public/vendor/visibility/visibility.min.js',
- 'node_modules/bootstrap/dist/js/bootstrap.js',
- 'public/vendor/jquery/bootstrap-tagsinput/bootstrap-tagsinput.min.js',
- 'public/vendor/jquery/textcomplete/jquery.textcomplete.js',
- 'public/vendor/requirejs/require.js',
- 'public/src/require-config.js',
- 'public/vendor/bootbox/bootbox.js',
- 'public/vendor/bootbox/wrapper.js',
- 'public/vendor/tinycon/tinycon.js',
- 'public/vendor/xregexp/xregexp.js',
- 'public/vendor/xregexp/unicode/unicode-base.js',
- 'node_modules/templates.js/lib/templates.js',
- 'public/src/utils.js',
- 'public/src/sockets.js',
- 'public/src/app.js',
- 'public/src/ajaxify.js',
- 'public/src/overrides.js',
- 'public/src/widgets.js',
- 'node_modules/promise-polyfill/promise.js',
- ],
-
- // files listed below are only available client-side, or are bundled in to reduce # of network requests on cold load
- rjs: [
- 'public/src/client/footer.js',
- 'public/src/client/chats.js',
- 'public/src/client/infinitescroll.js',
- 'public/src/client/pagination.js',
- 'public/src/client/recent.js',
- 'public/src/client/unread.js',
- 'public/src/client/topic.js',
- 'public/src/client/topic/events.js',
- 'public/src/client/topic/fork.js',
- 'public/src/client/topic/move.js',
- 'public/src/client/topic/posts.js',
- 'public/src/client/topic/images.js',
- 'public/src/client/topic/postTools.js',
- 'public/src/client/topic/threadTools.js',
- 'public/src/client/categories.js',
- 'public/src/client/category.js',
- 'public/src/client/category/tools.js',
-
- 'public/src/modules/translator.js',
- 'public/src/modules/notifications.js',
- 'public/src/modules/chat.js',
- 'public/src/modules/components.js',
- 'public/src/modules/sort.js',
- 'public/src/modules/navigator.js',
- 'public/src/modules/topicSelect.js',
- 'public/src/modules/share.js',
- 'public/src/modules/search.js',
- 'public/src/modules/alerts.js',
- 'public/src/modules/taskbar.js',
- 'public/src/modules/helpers.js',
- 'public/src/modules/string.js',
- 'public/src/modules/flags.js',
- 'public/src/modules/storage.js',
- ],
-
- // modules listed below are built (/src/modules) so they can be defined anonymously
- modules: {
- 'Chart.js': 'node_modules/chart.js/dist/Chart.min.js',
- 'mousetrap.js': 'node_modules/mousetrap/mousetrap.min.js',
- 'cropper.js': 'node_modules/cropperjs/dist/cropper.min.js',
- 'jqueryui.js': 'public/vendor/jquery/js/jquery-ui.js',
- 'zxcvbn.js': 'node_modules/zxcvbn/dist/zxcvbn.js',
- ace: 'node_modules/ace-builds/src-min',
- },
+ Meta.js = {};
+
+ Meta.js.scripts = {
+ base: [
+ 'node_modules/jquery/dist/jquery.js',
+ 'node_modules/socket.io-client/dist/socket.io.js',
+ 'public/vendor/jquery/timeago/jquery.timeago.js',
+ 'public/vendor/jquery/js/jquery.form.min.js',
+ 'public/vendor/visibility/visibility.min.js',
+ 'node_modules/bootstrap/dist/js/bootstrap.js',
+ 'public/vendor/jquery/bootstrap-tagsinput/bootstrap-tagsinput.min.js',
+ 'public/vendor/jquery/textcomplete/jquery.textcomplete.js',
+ 'public/vendor/requirejs/require.js',
+ 'public/src/require-config.js',
+ 'public/vendor/bootbox/bootbox.js',
+ 'public/vendor/bootbox/wrapper.js',
+ 'public/vendor/tinycon/tinycon.js',
+ 'public/vendor/xregexp/xregexp.js',
+ 'public/vendor/xregexp/unicode/unicode-base.js',
+ 'node_modules/templates.js/lib/templates.js',
+ 'public/src/utils.js',
+ 'public/src/sockets.js',
+ 'public/src/app.js',
+ 'public/src/ajaxify.js',
+ 'public/src/overrides.js',
+ 'public/src/widgets.js',
+ 'node_modules/promise-polyfill/promise.js',
+ ],
+
+ // files listed below are only available client-side, or are bundled in to reduce # of network requests on cold load
+ rjs: [
+ 'public/src/client/footer.js',
+ 'public/src/client/chats.js',
+ 'public/src/client/infinitescroll.js',
+ 'public/src/client/pagination.js',
+ 'public/src/client/recent.js',
+ 'public/src/client/unread.js',
+ 'public/src/client/topic.js',
+ 'public/src/client/topic/events.js',
+ 'public/src/client/topic/fork.js',
+ 'public/src/client/topic/move.js',
+ 'public/src/client/topic/posts.js',
+ 'public/src/client/topic/images.js',
+ 'public/src/client/topic/postTools.js',
+ 'public/src/client/topic/threadTools.js',
+ 'public/src/client/categories.js',
+ 'public/src/client/category.js',
+ 'public/src/client/category/tools.js',
+
+ 'public/src/modules/translator.js',
+ 'public/src/modules/notifications.js',
+ 'public/src/modules/chat.js',
+ 'public/src/modules/components.js',
+ 'public/src/modules/sort.js',
+ 'public/src/modules/navigator.js',
+ 'public/src/modules/topicSelect.js',
+ 'public/src/modules/share.js',
+ 'public/src/modules/search.js',
+ 'public/src/modules/alerts.js',
+ 'public/src/modules/taskbar.js',
+ 'public/src/modules/helpers.js',
+ 'public/src/modules/string.js',
+ 'public/src/modules/flags.js',
+ 'public/src/modules/storage.js',
+ ],
+
+ // modules listed below are built (/src/modules) so they can be defined anonymously
+ modules: {
+ 'Chart.js': 'node_modules/chart.js/dist/Chart.min.js',
+ 'mousetrap.js': 'node_modules/mousetrap/mousetrap.min.js',
+ 'cropper.js': 'node_modules/cropperjs/dist/cropper.min.js',
+ 'jqueryui.js': 'public/vendor/jquery/js/jquery-ui.js',
+ 'zxcvbn.js': 'node_modules/zxcvbn/dist/zxcvbn.js',
+ ace: 'node_modules/ace-builds/src-min',
},
};
- function minifyModules(modules, callback) {
+ function minifyModules(modules, fork, callback) {
+ // for it to never fork
+ // otherwise it spawns way too many processes
+ // maybe eventually we can pool modules
+ // and pass the pools to the minifer
+ // to reduce the total number of threads
+ fork = false;
+
async.eachLimit(modules, 500, function (mod, next) {
var srcPath = mod.srcPath;
var destPath = mod.destPath;
- var minified;
- async.parallel([
- function (cb) {
+ async.parallel({
+ dirped: function (cb) {
mkdirp(path.dirname(destPath), cb);
},
- function (cb) {
+ minified: function (cb) {
fs.readFile(srcPath, function (err, buffer) {
if (err) {
return cb(err);
}
if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) {
- minified = { code: buffer.toString() };
- return cb();
- }
-
- try {
- minified = uglifyjs.minify(buffer.toString(), {
- fromString: true,
- compress: false,
- });
- } catch (e) {
- return cb(e);
+ return cb(null, { code: buffer.toString() });
}
- cb();
+ minifier.js.minify(buffer.toString(), fork, cb);
});
},
- ], function (err) {
+ }, function (err, results) {
if (err) {
return next(err);
}
+ var minified = results.minified;
fs.writeFile(destPath, minified.code, next);
});
}, callback);
@@ -233,7 +225,7 @@ module.exports = function (Meta) {
});
}
- Meta.js.buildModules = function (callback) {
+ Meta.js.buildModules = function (fork, callback) {
async.waterfall([
clearModules,
function (next) {
@@ -244,7 +236,7 @@ module.exports = function (Meta) {
getModuleList(next);
},
function (modules, next) {
- minifyModules(modules, next);
+ minifyModules(modules, fork, next);
},
], callback);
};
@@ -269,52 +261,13 @@ module.exports = function (Meta) {
});
};
- Meta.js.minify = function (target, callback) {
- winston.verbose('[meta/js] Minifying ' + target);
-
- var forkProcessParams = setupDebugging();
- var minifier = fork(minifierPath, [], forkProcessParams);
- Meta.js.minifierProc = minifier;
-
- Meta.js.target[target] = {};
-
- Meta.js.prepare(target, function (err) {
- if (err) {
- return callback(err);
- }
- minifier.send({
- action: 'js',
- minify: global.env !== 'development',
- scripts: Meta.js.target[target].scripts,
- });
- });
-
- minifier.on('message', function (message) {
- switch (message.type) {
- case 'end':
- Meta.js.target[target].cache = message.minified;
- Meta.js.target[target].map = message.sourceMap;
- winston.verbose('[meta/js] ' + target + ' minification complete');
- minifier.kill();
-
- Meta.js.commitToFile(target, callback);
- break;
- case 'error':
- winston.error('[meta/js] Could not compile ' + target + ': ' + message.message);
- minifier.kill();
-
- callback(new Error(message.message));
- break;
- }
- });
- };
-
- Meta.js.prepare = function (target, callback) {
- var pluginsScripts = [];
-
+ function getBundleScriptList(target, callback) {
var pluginDirectories = [];
- pluginsScripts = plugins[target === 'nodebb.min.js' ? 'clientScripts' : 'acpScripts'].filter(function (path) {
+ if (target === 'admin') {
+ target = 'acp';
+ }
+ var pluginScripts = plugins[target + 'Scripts'].filter(function (path) {
if (path.endsWith('.js')) {
return true;
}
@@ -325,8 +278,12 @@ module.exports = function (Meta) {
async.each(pluginDirectories, function (directory, next) {
file.walk(directory, function (err, scripts) {
- pluginsScripts = pluginsScripts.concat(scripts);
- next(err);
+ if (err) {
+ return next(err);
+ }
+
+ pluginScripts = pluginScripts.concat(scripts);
+ next();
});
}, function (err) {
if (err) {
@@ -335,52 +292,43 @@ module.exports = function (Meta) {
var basePath = path.resolve(__dirname, '../..');
- Meta.js.target[target].scripts = Meta.js.scripts.base.concat(pluginsScripts);
+ var scripts = Meta.js.scripts.base.concat(pluginScripts);
- if (target === 'nodebb.min.js') {
- Meta.js.target[target].scripts = Meta.js.target[target].scripts.concat(Meta.js.scripts.rjs);
+ if (target === 'client' && global.env !== 'development') {
+ scripts = scripts.concat(Meta.js.scripts.rjs);
}
- Meta.js.target[target].scripts = Meta.js.target[target].scripts.map(function (script) {
+ scripts = scripts.map(function (script) {
return path.resolve(basePath, script).replace(/\\/g, '/');
});
- callback();
+ callback(null, scripts);
});
- };
+ }
- Meta.js.killMinifier = function () {
- if (Meta.js.minifierProc) {
- Meta.js.minifierProc.kill('SIGTERM');
- }
- };
+ Meta.js.buildBundle = function (target, fork, callback) {
+ var fileNames = {
+ client: 'nodebb.min.js',
+ admin: 'acp.min.js',
+ };
- Meta.js.commitToFile = function (target, callback) {
- fs.writeFile(path.join(__dirname, '../../build/public', target), Meta.js.target[target].cache, function (err) {
- callback(err);
- });
- };
+ async.waterfall([
+ function (next) {
+ getBundleScriptList(target, next);
+ },
+ function (files, next) {
+ var minify = global.env !== 'development';
- function setupDebugging() {
- /**
- * Check if the parent process is running with the debug option --debug (or --debug-brk)
- */
- var forkProcessParams = {};
- if (global.v8debug || parseInt(process.execArgv.indexOf('--debug'), 10) !== -1) {
- /**
- * use the line below if you want to debug minifier.js script too (or even --debug-brk option, but
- * you'll have to setup your debugger and connect to the forked process)
- */
- // forkProcessParams = {execArgv: ['--debug=' + (global.process.debugPort + 1), '--nolazy']};
-
- /**
- * otherwise, just clean up --debug/--debug-brk options which are set up by default from the parent one
- */
- forkProcessParams = {
- execArgv: [],
- };
- }
+ minifier.js.bundle(files, minify, fork, next);
+ },
+ function (bundle, next) {
+ var filePath = path.join(__dirname, '../../build/public', fileNames[target]);
+ fs.writeFile(filePath, bundle.code, next);
+ },
+ ], callback);
+ };
- return forkProcessParams;
- }
+ Meta.js.killMinifier = function () {
+ minifier.killAll();
+ };
};
diff --git a/src/meta/minifier.js b/src/meta/minifier.js
index 43761c9d33..b1f2888b16 100644
--- a/src/meta/minifier.js
+++ b/src/meta/minifier.js
@@ -3,86 +3,255 @@
var uglifyjs = require('uglify-js');
var async = require('async');
var fs = require('fs');
+var childProcess = require('child_process');
+var os = require('os');
+var less = require('less');
+var postcss = require('postcss');
+var autoprefixer = require('autoprefixer');
+var clean = require('postcss-clean');
+
var file = require('../file');
-var Minifier = {
- js: {},
+var Minifier = module.exports;
+
+function setupDebugging() {
+ /**
+ * Check if the parent process is running with the debug option --debug (or --debug-brk)
+ */
+ var forkProcessParams = {};
+ if (global.v8debug || parseInt(process.execArgv.indexOf('--debug'), 10) !== -1) {
+ /**
+ * use the line below if you want to debug minifier.js script too (or even --debug-brk option, but
+ * you'll have to setup your debugger and connect to the forked process)
+ */
+ // forkProcessParams = { execArgv: ['--debug=' + (global.process.debugPort + 1), '--nolazy'] };
+
+ /**
+ * otherwise, just clean up --debug/--debug-brk options which are set up by default from the parent one
+ */
+ forkProcessParams = {
+ execArgv: [],
+ };
+ }
+
+ return forkProcessParams;
+}
+
+var children = [];
+
+Minifier.killAll = function () {
+ children.forEach(function (child) {
+ child.kill('SIGTERM');
+ });
+
+ children = [];
};
-/* Javascript */
-Minifier.js.minify = function (scripts, minify, callback) {
- scripts = scripts.filter(function (file) {
- return file && file.endsWith('.js');
+function removeChild(proc) {
+ children = children.filter(function (child) {
+ return child !== proc;
});
+}
- async.filter(scripts, function (script, next) {
- file.exists(script, function (err, exists) {
- if (err) {
- return next(err);
- }
+function forkAction(action, callback) {
+ var forkProcessParams = setupDebugging();
+ var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, {
+ cwd: __dirname,
+ env: {
+ minifier_child: true,
+ },
+ }));
- if (!exists) {
- console.warn('[minifier] file not found, ' + script);
- }
- next(null, exists);
- });
- }, function (err, scripts) {
- if (err) {
- return callback(err);
+ children.push(proc);
+
+ proc.on('message', function (message) {
+ if (message.type === 'error') {
+ proc.kill();
+ return callback(new Error(message.message));
}
- if (minify) {
- minifyScripts(scripts, callback);
- } else {
- concatenateScripts(scripts, callback);
+ if (message.type === 'end') {
+ proc.kill();
+ callback(null, message.result);
}
});
-};
+ proc.on('error', function (err) {
+ proc.kill();
+ removeChild(proc);
+ callback(err);
+ });
-process.on('message', function (payload) {
- switch (payload.action) {
- case 'js':
- Minifier.js.minify(payload.scripts, payload.minify, function (minified/* , sourceMap*/) {
- process.send({
- type: 'end',
- // sourceMap: sourceMap,
- minified: minified,
+ proc.send({
+ type: 'action',
+ action: action,
+ });
+
+ proc.on('close', function () {
+ removeChild(proc);
+ });
+}
+
+var actions = {};
+
+if (process.env.minifier_child) {
+ process.on('message', function (message) {
+ if (message.type === 'action') {
+ var action = message.action;
+ if (typeof actions[action.act] !== 'function') {
+ process.send({
+ type: 'error',
+ message: 'Unknown action',
+ });
+ return;
+ }
+
+ actions[action.act](action, function (err, result) {
+ if (err) {
+ process.send({
+ type: 'error',
+ message: err.message,
+ });
+ return;
+ }
+
+ process.send({
+ type: 'end',
+ result: result,
+ });
});
- });
- break;
+ }
+ });
+}
+
+function executeAction(action, fork, callback) {
+ if (fork) {
+ forkAction(action, callback);
+ } else {
+ if (typeof actions[action.act] !== 'function') {
+ return callback(Error('Unknown action'));
+ }
+ actions[action.act](action, callback);
}
-});
-
-function minifyScripts(scripts, callback) {
- // The portions of code involving the source map are commented out as they're broken in UglifyJS2
- // Follow along here: https://github.com/mishoo/UglifyJS2/issues/700
- try {
- var minified = uglifyjs.minify(scripts, {
- // outSourceMap: "nodebb.min.js.map",
- compress: false,
- });
+}
+
+function concat(data, callback) {
+ if (data.files && data.files.length) {
+ async.mapLimit(data.files, 1000, fs.readFile, function (err, files) {
+ if (err) {
+ return callback(err);
+ }
- callback(minified.code/* , minified.map*/);
- } catch (err) {
- process.send({
- type: 'error',
- message: err.message,
+ var output = files.join(os.EOL + ';');
+ callback(null, { code: output });
});
+
+ return;
}
+
+ callback();
}
+actions.concat = concat;
-function concatenateScripts(scripts, callback) {
- async.map(scripts, fs.readFile, function (err, scripts) {
- if (err) {
- process.send({
- type: 'error',
- message: err.message,
+function minifyJS(data, callback) {
+ var minified;
+
+ if (data.fromSource) {
+ var sources = data.source;
+ var multiple = Array.isArray(sources);
+ if (!multiple) {
+ sources = [sources];
+ }
+
+ try {
+ minified = sources.map(function (source) {
+ return uglifyjs.minify(source, {
+ // outSourceMap: data.filename + '.map',
+ compress: data.compress,
+ fromString: true,
+ output: {
+ // suppress uglify line length warnings
+ max_line_len: 400000,
+ },
+ });
});
- return;
+ } catch (e) {
+ return callback(e);
}
- scripts = scripts.join(require('os').EOL + ';');
+ return callback(null, multiple ? minified : minified[0]);
+ }
+
+ if (data.files && data.files.length) {
+ async.filter(data.files, file.exists, function (err, scripts) {
+ if (err) {
+ return callback(err);
+ }
+
+ try {
+ minified = uglifyjs.minify(scripts, {
+ // outSourceMap: data.filename + '.map',
+ compress: data.compress,
+ fromString: false,
+ });
+ } catch (e) {
+ return callback(e);
+ }
+
+ callback(null, minified);
+ });
- callback(scripts);
+ return;
+ }
+
+ callback();
+}
+actions.minifyJS = minifyJS;
+
+Minifier.js = {};
+Minifier.js.bundle = function (scripts, minify, fork, callback) {
+ executeAction({
+ act: minify ? 'minifyJS' : 'concat',
+ files: scripts,
+ compress: false,
+ }, fork, callback);
+};
+
+Minifier.js.minify = function (source, fork, callback) {
+ executeAction({
+ act: 'minifyJS',
+ fromSource: true,
+ source: source,
+ }, fork, callback);
+};
+
+function buildCSS(data, callback) {
+ less.render(data.source, {
+ paths: data.paths,
+ }, function (err, lessOutput) {
+ if (err) {
+ return callback(err);
+ }
+
+ postcss(data.minify ? [
+ autoprefixer,
+ clean({
+ processImportFrom: ['local'],
+ }),
+ ] : [autoprefixer]).process(lessOutput.css).then(function (result) {
+ callback(null, { code: result.css });
+ }, function (err) {
+ callback(err);
+ });
});
}
+actions.buildCSS = buildCSS;
+
+Minifier.css = {};
+Minifier.css.bundle = function (source, paths, minify, fork, callback) {
+ executeAction({
+ act: 'buildCSS',
+ source: source,
+ paths: paths,
+ minify: minify,
+ }, fork, callback);
+};
diff --git a/test/database/keys.js b/test/database/keys.js
index 157cc2ca97..afbf3c947c 100644
--- a/test/database/keys.js
+++ b/test/database/keys.js
@@ -102,6 +102,37 @@ describe('Key methods', function () {
});
});
+ it('should delete all sorted set elements', function (done) {
+ async.parallel([
+ function (next) {
+ db.sortedSetAdd('deletezset', 1, 'value1', next);
+ },
+ function (next) {
+ db.sortedSetAdd('deletezset', 2, 'value2', next);
+ },
+ ], function (err) {
+ if (err) {
+ return done(err);
+ }
+ db.delete('deletezset', function (err) {
+ assert.ifError(err);
+ async.parallel({
+ key1exists: function (next) {
+ db.isSortedSetMember('deletezset', 'value1', next);
+ },
+ key2exists: function (next) {
+ db.isSortedSetMember('deletezset', 'value2', next);
+ },
+ }, function (err, results) {
+ assert.equal(err, null);
+ assert.equal(results.key1exists, false);
+ assert.equal(results.key2exists, false);
+ done();
+ });
+ });
+ });
+ });
+
describe('increment', function () {
it('should initialize key to 1', function (done) {
db.increment('keyToIncrement', function (err, value) {