From 7ccfb5cdc02faf1158def95c51adedde383df8ee Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Wed, 17 May 2017 16:33:31 -0600 Subject: [PATCH] Build refactor - Target aliases and bundles - More concurrency - Use `--series` to force series processing - Minifier self-hosted threading - meta/css delegates to minifier --- package.json | 3 +- public/less/admin/admin.less | 2 +- src/meta/build.js | 283 +++++++++++++++++++------------ src/meta/css.js | 163 +++++++++--------- src/meta/js.js | 314 +++++++++++++++-------------------- src/meta/minifier.js | 285 ++++++++++++++++++++++++------- 6 files changed, 612 insertions(+), 438 deletions(-) diff --git a/package.json b/package.json index dff73ec5d5..791c5f36b8 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,8 @@ "coveralls": "istanbul cover _mocha --report lcovonly -- -R dot && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage" }, "dependencies": { - "async": "2.4.0", "ace-builds": "^1.2.6", + "async": "2.4.0", "autoprefixer": "7.0.1", "bcryptjs": "2.4.3", "body-parser": "^1.9.0", @@ -45,6 +45,7 @@ "jquery": "^3.1.0", "json-2-csv": "^2.0.22", "less": "^2.0.0", + "lodash.padstart": "^4.6.1", "logrotate-stream": "^0.2.3", "lru-cache": "4.0.2", "mime": "^1.3.4", diff --git a/public/less/admin/admin.less b/public/less/admin/admin.less index a37682ab83..8f713028ed 100644 --- a/public/less/admin/admin.less +++ b/public/less/admin/admin.less @@ -1,4 +1,4 @@ -@import "../../node_modules/bootstrap/less/bootstrap"; +@import "bootstrap/less/bootstrap"; @import "./paper/variables"; @import "./paper/bootswatch"; @import "./mixins"; diff --git a/src/meta/build.js b/src/meta/build.js index 9ba5ec89bf..e88bbb17e9 100644 --- a/src/meta/build.js +++ b/src/meta/build.js @@ -2,149 +2,212 @@ var async = require('async'); var winston = require('winston'); +var os = require('os'); +var nconf = require('nconf'); +var padstart = require('lodash.padstart'); -var buildStart; +var cacheBuster = require('./cacheBuster'); +var meta; -var valid = ['js', 'clientCSS', 'acpCSS', 'tpl', 'lang', 'sound']; +function step(target, callback) { + var startTime = Date.now(); + winston.info('[build] ' + target + ' build started'); -exports.buildAll = function (callback) { - exports.build(valid.join(','), callback); -}; + return function (err) { + if (err) { + winston.error('[build] ' + target + ' build failed'); + return callback(err); + } -exports.build = function build(targets, callback) { - buildStart = Date.now(); + var time = (Date.now() - startTime) / 1000; - var db = require('../database'); - var meta = require('../meta'); - var plugins = require('../plugins'); + winston.info('[build] ' + target + ' build completed in ' + time + 'sec'); + callback(); + }; +} + +var targetHandlers = { + 'plugin static dirs': function (parallel, callback) { + meta.js.linkStatics(callback); + }, + 'requirejs modules': function (parallel, callback) { + meta.js.buildModules(parallel, callback); + }, + 'client js bundle': function (parallel, callback) { + meta.js.buildBundle('client', parallel, callback); + }, + 'admin js bundle': function (parallel, callback) { + meta.js.buildBundle('admin', parallel, callback); + }, + javascript: [ + 'plugin static dirs', + 'requirejs modules', + 'client js bundle', + 'admin js bundle', + ], + 'client side styles': function (parallel, callback) { + meta.css.buildBundle('client', parallel, callback); + }, + 'admin control panel styles': function (parallel, callback) { + meta.css.buildBundle('admin', parallel, callback); + }, + styles: [ + 'client side styles', + 'admin control panel styles', + ], + templates: function (parallel, callback) { + meta.templates.compile(callback); + }, + languages: function (parallel, callback) { + meta.languages.build(callback); + }, + sounds: function (parallel, callback) { + meta.sounds.build(callback); + }, +}; +var aliases = { + 'plugin static dirs': ['staticdirs'], + 'requirejs modules': ['rjs', 'modules'], + 'client js bundle': ['clientjs', 'clientscript', 'clientscripts'], + 'admin js bundle': ['adminjs', 'adminscript', 'adminscripts'], + javascript: ['js'], + 'client side styles': [ + 'clientcss', 'clientless', 'clientstyles', 'clientstyle', + ], + 'admin control panel styles': [ + 'admincss', 'adminless', 'adminstyles', 'adminstyle', 'acpcss', 'acpless', 'acpstyles', 'acpstyle', + ], + styles: ['css', 'less', 'style'], + templates: ['tpl'], + languages: ['lang', 'i18n'], + sounds: ['sound'], +}; - targets = (targets === true ? valid : targets.split(',').filter(function (target) { - return valid.indexOf(target) !== -1; - })); +aliases = Object.keys(aliases).reduce(function (prev, key) { + var arr = aliases[key]; + arr.forEach(function (alias) { + prev[alias] = key; + }); + prev[key] = key; + return prev; +}, {}); - if (!targets) { - winston.error('[build] No valid build targets found. Aborting.'); - return process.exit(0); - } +function beforeBuild(callback) { + var db = require('../database'); + var plugins = require('../plugins'); + meta = require('../meta'); async.series([ - async.apply(db.init), - async.apply(meta.themes.setupPaths), - async.apply(plugins.prepareForBuild), + db.init, + meta.themes.setupPaths, + plugins.prepareForBuild, ], function (err) { if (err) { winston.error('[build] Encountered error preparing for build: ' + err.message); - return process.exit(1); + return callback(err); } - exports.buildTargets(targets, callback); + callback(); }); -}; +} -exports.buildTargets = function (targets, callback) { - var cacheBuster = require('./cacheBuster'); - var meta = require('../meta'); - var numCpus = require('os').cpus().length; - var parallel = targets.length > 1 && numCpus > 1; +var allTargets = Object.keys(targetHandlers).filter(function (name) { + return typeof targetHandlers[name] === 'function'; +}); +function buildTargets(targets, parallel, callback) { + var all = parallel ? async.each : async.eachSeries; - buildStart = buildStart || Date.now(); + var length = Math.max.apply(Math, targets.map(function (name) { + return name.length; + })); - var step = function (startTime, target, next, err) { - if (err) { - winston.error('Build failed: ' + err.stack); - process.exit(1); - } - winston.info('[build] ' + target + ' => Completed in ' + ((Date.now() - startTime) / 1000) + 's'); - next(); - }; + all(targets, function (target, next) { + targetHandlers[target](parallel, step(padstart(target, length) + ' ', next)); + }, callback); +} - if (parallel) { - winston.verbose('[build] Utilising multiple cores/processes'); - } else { - winston.verbose('[build] Utilising single-core'); +function build(targets, callback) { + if (targets === true) { + targets = allTargets; + } else if (!Array.isArray(targets)) { + targets = targets.split(','); } - async[parallel ? 'parallel' : 'series']([ + targets = targets + // get full target name + .map(function (target) { + target = target.toLowerCase().replace(/-/g, ''); + if (!aliases[target]) { + winston.warn('[build] Unknown target: ' + target); + return false; + } + + return aliases[target]; + }) + // filter nonexistent targets + .filter(Boolean) + // map multitargets to their sets + .reduce(function (prev, target) { + if (Array.isArray(targetHandlers[target])) { + return prev.concat(targetHandlers[target]); + } + + return prev.concat(target); + }, []) + // unique + .filter(function (target, i, arr) { + return arr.indexOf(target) === i; + }); + + if (typeof callback !== 'function') { + callback = function (err) { + if (err) { + winston.error(err); + process.exit(1); + } else { + process.exit(0); + } + }; + } + + if (!targets) { + winston.info('[build] No valid targets supplied. Aborting.'); + callback(); + } + + var startTime; + var totalTime; + async.series([ + beforeBuild, function (next) { - if (targets.indexOf('js') !== -1) { - winston.info('[build] Building javascript'); - var startTime = Date.now(); - async.series([ - meta.js.buildModules, - meta.js.linkStatics, - async.apply(meta.js.minify, 'nodebb.min.js'), - async.apply(meta.js.minify, 'acp.min.js'), - ], step.bind(this, startTime, 'js', next)); + var parallel = os.cpus().length > 1 && !nconf.get('series'); + if (parallel) { + winston.info('[build] Building in parallel mode'); } else { - setImmediate(next); + winston.info('[build] Building in series mode'); } + + startTime = Date.now(); + buildTargets(targets, parallel, next); }, function (next) { - async.eachSeries(targets, function (target, next) { - var startTime; - switch (target) { - case 'js': - setImmediate(next); - break; - case 'clientCSS': - winston.info('[build] Building client-side CSS'); - startTime = Date.now(); - meta.css.minify('client', step.bind(this, startTime, target, next)); - break; - - case 'acpCSS': - winston.info('[build] Building admin control panel CSS'); - startTime = Date.now(); - meta.css.minify('admin', step.bind(this, startTime, target, next)); - break; - - case 'tpl': - winston.info('[build] Building templates'); - startTime = Date.now(); - meta.templates.compile(step.bind(this, startTime, target, next)); - break; - - case 'lang': - winston.info('[build] Building language files'); - startTime = Date.now(); - meta.languages.build(step.bind(this, startTime, target, next)); - break; - - case 'sound': - winston.info('[build] Linking sound files'); - startTime = Date.now(); - meta.sounds.build(step.bind(this, startTime, target, next)); - break; - - default: - winston.warn('[build] Unknown build target: \'' + target + '\''); - setImmediate(next); - break; - } - }, next); + totalTime = (Date.now() - startTime) / 1000; + cacheBuster.write(next); }, ], function (err) { if (err) { winston.error('[build] Encountered error during build step: ' + err.message); - return process.exit(1); + return callback(err); } - cacheBuster.write(function (err) { - if (err) { - winston.error('[build] Failed to write `cache-buster.conf`: ' + err.message); - return process.exit(1); - } - - var time = (Date.now() - buildStart) / 1000; + winston.info('[build] Asset compilation successful. Completed in ' + totalTime + 'sec.'); + callback(); + }); +} - winston.info('[build] Asset compilation successful. Completed in ' + time + 's.'); +exports.build = build; - if (typeof callback === 'function') { - callback(); - } else { - process.exit(0); - } - }); - }); +exports.buildAll = function (callback) { + build(allTargets, callback); }; diff --git a/src/meta/css.js b/src/meta/css.js index 88ad3348e8..127191437c 100644 --- a/src/meta/css.js +++ b/src/meta/css.js @@ -4,15 +4,12 @@ var winston = require('winston'); var nconf = require('nconf'); var fs = require('fs'); var path = require('path'); -var less = require('less'); var async = require('async'); -var autoprefixer = require('autoprefixer'); -var postcss = require('postcss'); -var clean = require('postcss-clean'); var plugins = require('../plugins'); var db = require('../database'); var file = require('../file'); +var minifier = require('./minifier'); module.exports = function (Meta) { Meta.css = {}; @@ -49,50 +46,19 @@ module.exports = function (Meta) { }, }; - Meta.css.minify = function (target, callback) { - callback = callback || function () {}; - - winston.verbose('[meta/css] Minifying LESS/CSS'); - db.getObjectFields('config', ['theme:type', 'theme:id'], function (err, themeData) { - if (err) { - return callback(err); - } - - var themeId = (themeData['theme:id'] || 'nodebb-theme-persona'); - var baseThemePath = path.join(nconf.get('themes_path'), (themeData['theme:type'] && themeData['theme:type'] === 'local' ? themeId : 'nodebb-theme-vanilla')); - var paths = [ - baseThemePath, - path.join(__dirname, '../../node_modules'), - path.join(__dirname, '../../public/vendor/fontawesome/less'), - ]; - var source = ''; - - var lessFiles = filterMissingFiles(plugins.lessFiles); - var cssFiles = filterMissingFiles(plugins.cssFiles); - - async.waterfall([ - function (next) { - getStyleSource(cssFiles, '\n@import (inline) ".', '.css', next); - }, - function (src, next) { - source += src; - getStyleSource(lessFiles, '\n@import ".', '.less', next); - }, - function (src, next) { - source += src; - next(); - }, - ], function (err) { - if (err) { - return callback(err); + function filterMissingFiles(filepaths, callback) { + async.filter(filepaths, function (filepath, next) { + file.exists(path.join(__dirname, '../../node_modules', filepath), function (err, exists) { + if (!exists) { + winston.warn('[meta/css] File not found! ' + filepath); } - minify(buildImports[target](source), paths, target, callback); + next(err, exists); }); - }); - }; + }, callback); + } - function getStyleSource(files, prefix, extension, callback) { + function getImports(files, prefix, extension, callback) { var pluginDirectories = []; var source = ''; @@ -121,55 +87,82 @@ module.exports = function (Meta) { }); } - Meta.css.commitToFile = function (target, source, callback) { - var filename = (target === 'client' ? 'stylesheet' : 'admin') + '.css'; + function getBundleMetadata(target, callback) { + var paths = [ + path.join(__dirname, '../../node_modules'), + path.join(__dirname, '../../public/vendor/fontawesome/less'), + ]; - fs.writeFile(path.join(__dirname, '../../build/public/' + filename), source, function (err) { - if (!err) { - winston.verbose('[meta/css] ' + target + ' CSS committed to disk.'); - } else { - winston.error('[meta/css] ' + err.message); - process.exit(1); - } + async.waterfall([ + function (next) { + if (target !== 'client') { + return next(null, null); + } - callback(); - }); - }; + db.getObjectFields('config', ['theme:type', 'theme:id'], next); + }, + function (themeData, next) { + if (target === 'client') { + var themeId = (themeData['theme:id'] || 'nodebb-theme-persona'); + var baseThemePath = path.join(nconf.get('themes_path'), (themeData['theme:type'] && themeData['theme:type'] === 'local' ? themeId : 'nodebb-theme-vanilla')); + paths.unshift(baseThemePath); + } - function minify(source, paths, target, callback) { - callback = callback || function () {}; - less.render(source, { - paths: paths, - }, function (err, lessOutput) { + async.parallel({ + less: function (cb) { + async.waterfall([ + function (next) { + filterMissingFiles(plugins.lessFiles, next); + }, + function (lessFiles, next) { + getImports(lessFiles, '\n@import ".', '.less', next); + }, + ], cb); + }, + css: function (cb) { + async.waterfall([ + function (next) { + filterMissingFiles(plugins.cssFiles, next); + }, + function (cssFiles, next) { + getImports(cssFiles, '\n@import (inline) ".', '.css', next); + }, + ], cb); + }, + }, next); + }, + function (result, next) { + var cssImports = result.css; + var lessImports = result.less; + + var imports = cssImports + '\n' + lessImports; + imports = buildImports[target](imports); + + next(null, imports); + }, + ], function (err, imports) { if (err) { - winston.error('[meta/css] Could not minify LESS/CSS: ' + err.message); return callback(err); } - postcss(global.env === 'development' ? [autoprefixer] : [ - autoprefixer, - clean({ - processImportFrom: ['local'], - }), - ]).process(lessOutput.css).then(function (result) { - result.warnings().forEach(function (warn) { - winston.verbose(warn.toString()); - }); - - return Meta.css.commitToFile(target, result.css, function () { - callback(null, result.css); - }); - }); + callback(null, { paths: paths, imports: imports }); }); } - function filterMissingFiles(files) { - return files.filter(function (filePath) { - var exists = file.existsSync(path.join(__dirname, '../../node_modules', filePath)); - if (!exists) { - winston.warn('[meta/css] File not found! ' + filePath); - } - return exists; - }); - } + Meta.css.buildBundle = function (target, fork, callback) { + async.waterfall([ + function (next) { + getBundleMetadata(target, next); + }, + function (data, next) { + var minify = global.env !== 'development'; + minifier.css.bundle(data.imports, data.paths, minify, fork, next); + }, + function (bundle, next) { + var filename = (target === 'client' ? 'stylesheet' : 'admin') + '.css'; + + fs.writeFile(path.join(__dirname, '../../build/public', filename), bundle.code, next); + }, + ], callback); + }; }; diff --git a/src/meta/js.js b/src/meta/js.js index f654d45644..e7b22939fc 100644 --- a/src/meta/js.js +++ b/src/meta/js.js @@ -1,136 +1,128 @@ 'use strict'; -var winston = require('winston'); -var fork = require('child_process').fork; var path = require('path'); var async = require('async'); var fs = require('fs'); var mkdirp = require('mkdirp'); var rimraf = require('rimraf'); -var uglifyjs = require('uglify-js'); var file = require('../file'); var plugins = require('../plugins'); - -var minifierPath = path.join(__dirname, 'minifier.js'); +var minifier = require('./minifier'); module.exports = function (Meta) { - Meta.js = { - target: {}, - scripts: { - base: [ - 'node_modules/jquery/dist/jquery.js', - 'node_modules/socket.io-client/dist/socket.io.js', - 'public/vendor/jquery/timeago/jquery.timeago.js', - 'public/vendor/jquery/js/jquery.form.min.js', - 'public/vendor/visibility/visibility.min.js', - 'node_modules/bootstrap/dist/js/bootstrap.js', - 'public/vendor/jquery/bootstrap-tagsinput/bootstrap-tagsinput.min.js', - 'public/vendor/jquery/textcomplete/jquery.textcomplete.js', - 'public/vendor/requirejs/require.js', - 'public/src/require-config.js', - 'public/vendor/bootbox/bootbox.js', - 'public/vendor/bootbox/wrapper.js', - 'public/vendor/tinycon/tinycon.js', - 'public/vendor/xregexp/xregexp.js', - 'public/vendor/xregexp/unicode/unicode-base.js', - 'node_modules/templates.js/lib/templates.js', - 'public/src/utils.js', - 'public/src/sockets.js', - 'public/src/app.js', - 'public/src/ajaxify.js', - 'public/src/overrides.js', - 'public/src/widgets.js', - 'node_modules/promise-polyfill/promise.js', - ], - - // files listed below are only available client-side, or are bundled in to reduce # of network requests on cold load - rjs: [ - 'public/src/client/footer.js', - 'public/src/client/chats.js', - 'public/src/client/infinitescroll.js', - 'public/src/client/pagination.js', - 'public/src/client/recent.js', - 'public/src/client/unread.js', - 'public/src/client/topic.js', - 'public/src/client/topic/events.js', - 'public/src/client/topic/fork.js', - 'public/src/client/topic/move.js', - 'public/src/client/topic/posts.js', - 'public/src/client/topic/images.js', - 'public/src/client/topic/postTools.js', - 'public/src/client/topic/threadTools.js', - 'public/src/client/categories.js', - 'public/src/client/category.js', - 'public/src/client/category/tools.js', - - 'public/src/modules/translator.js', - 'public/src/modules/notifications.js', - 'public/src/modules/chat.js', - 'public/src/modules/components.js', - 'public/src/modules/sort.js', - 'public/src/modules/navigator.js', - 'public/src/modules/topicSelect.js', - 'public/src/modules/share.js', - 'public/src/modules/search.js', - 'public/src/modules/alerts.js', - 'public/src/modules/taskbar.js', - 'public/src/modules/helpers.js', - 'public/src/modules/string.js', - 'public/src/modules/flags.js', - 'public/src/modules/storage.js', - ], - - // modules listed below are built (/src/modules) so they can be defined anonymously - modules: { - 'Chart.js': 'node_modules/chart.js/dist/Chart.min.js', - 'mousetrap.js': 'node_modules/mousetrap/mousetrap.min.js', - 'cropper.js': 'node_modules/cropperjs/dist/cropper.min.js', - 'jqueryui.js': 'public/vendor/jquery/js/jquery-ui.js', - 'zxcvbn.js': 'node_modules/zxcvbn/dist/zxcvbn.js', - ace: 'node_modules/ace-builds/src-min', - }, + Meta.js = {}; + + Meta.js.scripts = { + base: [ + 'node_modules/jquery/dist/jquery.js', + 'node_modules/socket.io-client/dist/socket.io.js', + 'public/vendor/jquery/timeago/jquery.timeago.js', + 'public/vendor/jquery/js/jquery.form.min.js', + 'public/vendor/visibility/visibility.min.js', + 'node_modules/bootstrap/dist/js/bootstrap.js', + 'public/vendor/jquery/bootstrap-tagsinput/bootstrap-tagsinput.min.js', + 'public/vendor/jquery/textcomplete/jquery.textcomplete.js', + 'public/vendor/requirejs/require.js', + 'public/src/require-config.js', + 'public/vendor/bootbox/bootbox.js', + 'public/vendor/bootbox/wrapper.js', + 'public/vendor/tinycon/tinycon.js', + 'public/vendor/xregexp/xregexp.js', + 'public/vendor/xregexp/unicode/unicode-base.js', + 'node_modules/templates.js/lib/templates.js', + 'public/src/utils.js', + 'public/src/sockets.js', + 'public/src/app.js', + 'public/src/ajaxify.js', + 'public/src/overrides.js', + 'public/src/widgets.js', + 'node_modules/promise-polyfill/promise.js', + ], + + // files listed below are only available client-side, or are bundled in to reduce # of network requests on cold load + rjs: [ + 'public/src/client/footer.js', + 'public/src/client/chats.js', + 'public/src/client/infinitescroll.js', + 'public/src/client/pagination.js', + 'public/src/client/recent.js', + 'public/src/client/unread.js', + 'public/src/client/topic.js', + 'public/src/client/topic/events.js', + 'public/src/client/topic/fork.js', + 'public/src/client/topic/move.js', + 'public/src/client/topic/posts.js', + 'public/src/client/topic/images.js', + 'public/src/client/topic/postTools.js', + 'public/src/client/topic/threadTools.js', + 'public/src/client/categories.js', + 'public/src/client/category.js', + 'public/src/client/category/tools.js', + + 'public/src/modules/translator.js', + 'public/src/modules/notifications.js', + 'public/src/modules/chat.js', + 'public/src/modules/components.js', + 'public/src/modules/sort.js', + 'public/src/modules/navigator.js', + 'public/src/modules/topicSelect.js', + 'public/src/modules/share.js', + 'public/src/modules/search.js', + 'public/src/modules/alerts.js', + 'public/src/modules/taskbar.js', + 'public/src/modules/helpers.js', + 'public/src/modules/string.js', + 'public/src/modules/flags.js', + 'public/src/modules/storage.js', + ], + + // modules listed below are built (/src/modules) so they can be defined anonymously + modules: { + 'Chart.js': 'node_modules/chart.js/dist/Chart.min.js', + 'mousetrap.js': 'node_modules/mousetrap/mousetrap.min.js', + 'cropper.js': 'node_modules/cropperjs/dist/cropper.min.js', + 'jqueryui.js': 'public/vendor/jquery/js/jquery-ui.js', + 'zxcvbn.js': 'node_modules/zxcvbn/dist/zxcvbn.js', + ace: 'node_modules/ace-builds/src-min', }, }; - function minifyModules(modules, callback) { + function minifyModules(modules, fork, callback) { + // for it to never fork + // otherwise it spawns way too many processes + // maybe eventually we can pool modules + // and pass the pools to the minifer + // to reduce the total number of threads + fork = false; + async.eachLimit(modules, 500, function (mod, next) { var srcPath = mod.srcPath; var destPath = mod.destPath; - var minified; - async.parallel([ - function (cb) { + async.parallel({ + dirped: function (cb) { mkdirp(path.dirname(destPath), cb); }, - function (cb) { + minified: function (cb) { fs.readFile(srcPath, function (err, buffer) { if (err) { return cb(err); } if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) { - minified = { code: buffer.toString() }; - return cb(); - } - - try { - minified = uglifyjs.minify(buffer.toString(), { - fromString: true, - compress: false, - }); - } catch (e) { - return cb(e); + return cb(null, { code: buffer.toString() }); } - cb(); + minifier.js.minify(buffer.toString(), fork, cb); }); }, - ], function (err) { + }, function (err, results) { if (err) { return next(err); } + var minified = results.minified; fs.writeFile(destPath, minified.code, next); }); }, callback); @@ -233,7 +225,7 @@ module.exports = function (Meta) { }); } - Meta.js.buildModules = function (callback) { + Meta.js.buildModules = function (fork, callback) { async.waterfall([ clearModules, function (next) { @@ -244,7 +236,7 @@ module.exports = function (Meta) { getModuleList(next); }, function (modules, next) { - minifyModules(modules, next); + minifyModules(modules, fork, next); }, ], callback); }; @@ -269,52 +261,13 @@ module.exports = function (Meta) { }); }; - Meta.js.minify = function (target, callback) { - winston.verbose('[meta/js] Minifying ' + target); - - var forkProcessParams = setupDebugging(); - var minifier = fork(minifierPath, [], forkProcessParams); - Meta.js.minifierProc = minifier; - - Meta.js.target[target] = {}; - - Meta.js.prepare(target, function (err) { - if (err) { - return callback(err); - } - minifier.send({ - action: 'js', - minify: global.env !== 'development', - scripts: Meta.js.target[target].scripts, - }); - }); - - minifier.on('message', function (message) { - switch (message.type) { - case 'end': - Meta.js.target[target].cache = message.minified; - Meta.js.target[target].map = message.sourceMap; - winston.verbose('[meta/js] ' + target + ' minification complete'); - minifier.kill(); - - Meta.js.commitToFile(target, callback); - break; - case 'error': - winston.error('[meta/js] Could not compile ' + target + ': ' + message.message); - minifier.kill(); - - callback(new Error(message.message)); - break; - } - }); - }; - - Meta.js.prepare = function (target, callback) { - var pluginsScripts = []; - + function getBundleScriptList(target, callback) { var pluginDirectories = []; - pluginsScripts = plugins[target === 'nodebb.min.js' ? 'clientScripts' : 'acpScripts'].filter(function (path) { + if (target === 'admin') { + target = 'acp'; + } + var pluginScripts = plugins[target + 'Scripts'].filter(function (path) { if (path.endsWith('.js')) { return true; } @@ -325,8 +278,12 @@ module.exports = function (Meta) { async.each(pluginDirectories, function (directory, next) { file.walk(directory, function (err, scripts) { - pluginsScripts = pluginsScripts.concat(scripts); - next(err); + if (err) { + return next(err); + } + + pluginScripts = pluginScripts.concat(scripts); + next(); }); }, function (err) { if (err) { @@ -335,52 +292,43 @@ module.exports = function (Meta) { var basePath = path.resolve(__dirname, '../..'); - Meta.js.target[target].scripts = Meta.js.scripts.base.concat(pluginsScripts); + var scripts = Meta.js.scripts.base.concat(pluginScripts); - if (target === 'nodebb.min.js') { - Meta.js.target[target].scripts = Meta.js.target[target].scripts.concat(Meta.js.scripts.rjs); + if (target === 'client' && global.env !== 'development') { + scripts = scripts.concat(Meta.js.scripts.rjs); } - Meta.js.target[target].scripts = Meta.js.target[target].scripts.map(function (script) { + scripts = scripts.map(function (script) { return path.resolve(basePath, script).replace(/\\/g, '/'); }); - callback(); + callback(null, scripts); }); - }; + } - Meta.js.killMinifier = function () { - if (Meta.js.minifierProc) { - Meta.js.minifierProc.kill('SIGTERM'); - } - }; + Meta.js.buildBundle = function (target, fork, callback) { + var fileNames = { + client: 'nodebb.min.js', + admin: 'acp.min.js', + }; - Meta.js.commitToFile = function (target, callback) { - fs.writeFile(path.join(__dirname, '../../build/public', target), Meta.js.target[target].cache, function (err) { - callback(err); - }); - }; + async.waterfall([ + function (next) { + getBundleScriptList(target, next); + }, + function (files, next) { + var minify = global.env !== 'development'; - function setupDebugging() { - /** - * Check if the parent process is running with the debug option --debug (or --debug-brk) - */ - var forkProcessParams = {}; - if (global.v8debug || parseInt(process.execArgv.indexOf('--debug'), 10) !== -1) { - /** - * use the line below if you want to debug minifier.js script too (or even --debug-brk option, but - * you'll have to setup your debugger and connect to the forked process) - */ - // forkProcessParams = {execArgv: ['--debug=' + (global.process.debugPort + 1), '--nolazy']}; - - /** - * otherwise, just clean up --debug/--debug-brk options which are set up by default from the parent one - */ - forkProcessParams = { - execArgv: [], - }; - } + minifier.js.bundle(files, minify, fork, next); + }, + function (bundle, next) { + var filePath = path.join(__dirname, '../../build/public', fileNames[target]); + fs.writeFile(filePath, bundle.code, next); + }, + ], callback); + }; - return forkProcessParams; - } + Meta.js.killMinifier = function () { + minifier.killAll(); + }; }; diff --git a/src/meta/minifier.js b/src/meta/minifier.js index 43761c9d33..b1f2888b16 100644 --- a/src/meta/minifier.js +++ b/src/meta/minifier.js @@ -3,86 +3,255 @@ var uglifyjs = require('uglify-js'); var async = require('async'); var fs = require('fs'); +var childProcess = require('child_process'); +var os = require('os'); +var less = require('less'); +var postcss = require('postcss'); +var autoprefixer = require('autoprefixer'); +var clean = require('postcss-clean'); + var file = require('../file'); -var Minifier = { - js: {}, +var Minifier = module.exports; + +function setupDebugging() { + /** + * Check if the parent process is running with the debug option --debug (or --debug-brk) + */ + var forkProcessParams = {}; + if (global.v8debug || parseInt(process.execArgv.indexOf('--debug'), 10) !== -1) { + /** + * use the line below if you want to debug minifier.js script too (or even --debug-brk option, but + * you'll have to setup your debugger and connect to the forked process) + */ + // forkProcessParams = { execArgv: ['--debug=' + (global.process.debugPort + 1), '--nolazy'] }; + + /** + * otherwise, just clean up --debug/--debug-brk options which are set up by default from the parent one + */ + forkProcessParams = { + execArgv: [], + }; + } + + return forkProcessParams; +} + +var children = []; + +Minifier.killAll = function () { + children.forEach(function (child) { + child.kill('SIGTERM'); + }); + + children = []; }; -/* Javascript */ -Minifier.js.minify = function (scripts, minify, callback) { - scripts = scripts.filter(function (file) { - return file && file.endsWith('.js'); +function removeChild(proc) { + children = children.filter(function (child) { + return child !== proc; }); +} - async.filter(scripts, function (script, next) { - file.exists(script, function (err, exists) { - if (err) { - return next(err); - } +function forkAction(action, callback) { + var forkProcessParams = setupDebugging(); + var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, { + cwd: __dirname, + env: { + minifier_child: true, + }, + })); - if (!exists) { - console.warn('[minifier] file not found, ' + script); - } - next(null, exists); - }); - }, function (err, scripts) { - if (err) { - return callback(err); + children.push(proc); + + proc.on('message', function (message) { + if (message.type === 'error') { + proc.kill(); + return callback(new Error(message.message)); } - if (minify) { - minifyScripts(scripts, callback); - } else { - concatenateScripts(scripts, callback); + if (message.type === 'end') { + proc.kill(); + callback(null, message.result); } }); -}; + proc.on('error', function (err) { + proc.kill(); + removeChild(proc); + callback(err); + }); -process.on('message', function (payload) { - switch (payload.action) { - case 'js': - Minifier.js.minify(payload.scripts, payload.minify, function (minified/* , sourceMap*/) { - process.send({ - type: 'end', - // sourceMap: sourceMap, - minified: minified, + proc.send({ + type: 'action', + action: action, + }); + + proc.on('close', function () { + removeChild(proc); + }); +} + +var actions = {}; + +if (process.env.minifier_child) { + process.on('message', function (message) { + if (message.type === 'action') { + var action = message.action; + if (typeof actions[action.act] !== 'function') { + process.send({ + type: 'error', + message: 'Unknown action', + }); + return; + } + + actions[action.act](action, function (err, result) { + if (err) { + process.send({ + type: 'error', + message: err.message, + }); + return; + } + + process.send({ + type: 'end', + result: result, + }); }); - }); - break; + } + }); +} + +function executeAction(action, fork, callback) { + if (fork) { + forkAction(action, callback); + } else { + if (typeof actions[action.act] !== 'function') { + return callback(Error('Unknown action')); + } + actions[action.act](action, callback); } -}); - -function minifyScripts(scripts, callback) { - // The portions of code involving the source map are commented out as they're broken in UglifyJS2 - // Follow along here: https://github.com/mishoo/UglifyJS2/issues/700 - try { - var minified = uglifyjs.minify(scripts, { - // outSourceMap: "nodebb.min.js.map", - compress: false, - }); +} + +function concat(data, callback) { + if (data.files && data.files.length) { + async.mapLimit(data.files, 1000, fs.readFile, function (err, files) { + if (err) { + return callback(err); + } - callback(minified.code/* , minified.map*/); - } catch (err) { - process.send({ - type: 'error', - message: err.message, + var output = files.join(os.EOL + ';'); + callback(null, { code: output }); }); + + return; } + + callback(); } +actions.concat = concat; -function concatenateScripts(scripts, callback) { - async.map(scripts, fs.readFile, function (err, scripts) { - if (err) { - process.send({ - type: 'error', - message: err.message, +function minifyJS(data, callback) { + var minified; + + if (data.fromSource) { + var sources = data.source; + var multiple = Array.isArray(sources); + if (!multiple) { + sources = [sources]; + } + + try { + minified = sources.map(function (source) { + return uglifyjs.minify(source, { + // outSourceMap: data.filename + '.map', + compress: data.compress, + fromString: true, + output: { + // suppress uglify line length warnings + max_line_len: 400000, + }, + }); }); - return; + } catch (e) { + return callback(e); } - scripts = scripts.join(require('os').EOL + ';'); + return callback(null, multiple ? minified : minified[0]); + } + + if (data.files && data.files.length) { + async.filter(data.files, file.exists, function (err, scripts) { + if (err) { + return callback(err); + } + + try { + minified = uglifyjs.minify(scripts, { + // outSourceMap: data.filename + '.map', + compress: data.compress, + fromString: false, + }); + } catch (e) { + return callback(e); + } + + callback(null, minified); + }); - callback(scripts); + return; + } + + callback(); +} +actions.minifyJS = minifyJS; + +Minifier.js = {}; +Minifier.js.bundle = function (scripts, minify, fork, callback) { + executeAction({ + act: minify ? 'minifyJS' : 'concat', + files: scripts, + compress: false, + }, fork, callback); +}; + +Minifier.js.minify = function (source, fork, callback) { + executeAction({ + act: 'minifyJS', + fromSource: true, + source: source, + }, fork, callback); +}; + +function buildCSS(data, callback) { + less.render(data.source, { + paths: data.paths, + }, function (err, lessOutput) { + if (err) { + return callback(err); + } + + postcss(data.minify ? [ + autoprefixer, + clean({ + processImportFrom: ['local'], + }), + ] : [autoprefixer]).process(lessOutput.css).then(function (result) { + callback(null, { code: result.css }); + }, function (err) { + callback(err); + }); }); } +actions.buildCSS = buildCSS; + +Minifier.css = {}; +Minifier.css.bundle = function (source, paths, minify, fork, callback) { + executeAction({ + act: 'buildCSS', + source: source, + paths: paths, + minify: minify, + }, fork, callback); +};