feat: improve grunt restart/rebuild speed

Gruntfile.js

@@ -1,95 +1,31 @@
 'use strict';
 
-
+const path = require('path');
-var async = require('async');
+const nconf = require('nconf');
-var fork = require('child_process').fork;
+nconf.argv().env({
-var env = process.env;
+	separator: '__',
+});
+const winston = require('winston');
+const fork = require('child_process').fork;
+const env = process.env;
 var worker;
-var updateWorker;
-var initWorker;
-var incomplete = [];
-var running = 0;
 
 env.NODE_ENV = env.NODE_ENV || 'development';
 
+const configFile = path.resolve(__dirname, nconf.any(['config', 'CONFIG']) || 'config.json');
+const prestart = require('./src/prestart');
+prestart.loadConfig(configFile);
 
-var nconf = require('nconf');
-nconf.file({
-	file: 'config.json',
-});
-
-nconf.defaults({
-	base_dir: __dirname,
-	views_dir: './build/public/templates',
-});
-var winston = require('winston');
-winston.configure({
-	transports: [
-		new winston.transports.Console({
-			handleExceptions: true,
-		}),
-	],
-});
 var db = require('./src/database');
 
 module.exports = function (grunt) {
 	var args = [];
-	var initArgs = ['--build'];
+
 	if (!grunt.option('verbose')) {
 		args.push('--log-level=info');
-		initArgs.push('--log-level=info');
+		nconf.set('log-level', 'info');
-	}
-
-	function update(action, filepath, target) {
-		var updateArgs = args.slice();
-		var compiling;
-		var time = Date.now();
-
-		if (target === 'lessUpdated_Client') {
-			compiling = 'clientCSS';
-		} else if (target === 'lessUpdated_Admin') {
-			compiling = 'acpCSS';
-		} else if (target === 'clientUpdated') {
-			compiling = 'js';
-		} else if (target === 'templatesUpdated') {
-			compiling = 'tpl';
-		} else if (target === 'langUpdated') {
-			compiling = 'lang';
-		} else if (target === 'serverUpdated') {
-			// Do nothing, just restart
-		}
-
-		if (compiling && !incomplete.includes(compiling)) {
-			incomplete.push(compiling);
-		}
-
-		updateArgs.push('--build');
-		updateArgs.push(incomplete.join(','));
-
-		worker.kill();
-		if (updateWorker) {
-			updateWorker.kill('SIGKILL');
-		}
-		updateWorker = fork('app.js', updateArgs, { env: env });
-		running += 1;
-		updateWorker.on('exit', function () {
-			running -= 1;
-			if (running === 0) {
-				worker = fork('app.js', args, {
-					env: env,
-				});
-				worker.on('message', function () {
-					if (incomplete.length) {
-						incomplete = [];
-
-						if (grunt.option('verbose')) {
-							grunt.log.writeln('NodeBB restarted in ' + (Date.now() - time) + ' ms');
-						}
-					}
-				});
-			}
-		});
 	}
+	prestart.setupWinston();
 
 	grunt.initConfig({
 		watch: {},
@@ -99,52 +35,49 @@ module.exports = function (grunt) {
 
 	grunt.registerTask('default', ['watch']);
 
-	grunt.registerTask('init', function () {
+	grunt.registerTask('init', async function () {
 		var done = this.async();
-		async.waterfall([
+		let plugins = [];
-			function (next) {
+		if (!process.argv.includes('--core')) {
-				db.init(next);
+			await db.init();
-			},
+			plugins = await db.getSortedSetRange('plugins:active', 0, -1);
-			function (next) {
+			addBaseThemes(plugins);
-				db.getSortedSetRange('plugins:active', 0, -1, next);
-			},
-			function (plugins, next) {
-				addBaseThemes(plugins, next);
-			},
-			function (plugins, next) {
 			if (!plugins.includes('nodebb-plugin-composer-default')) {
 				plugins.push('nodebb-plugin-composer-default');
 			}
-
-				if (process.argv.includes('--core')) {
-					plugins = [];
 		}
 
-				const lessUpdated_Client = plugins.map(p => 'node_modules/' + p + '/**/*.less');
+		const styleUpdated_Client = plugins.map(p => 'node_modules/' + p + '/*.less')
-				const lessUpdated_Admin = plugins.map(p => 'node_modules/' + p + '/**/*.less');
+			.concat(plugins.map(p => 'node_modules/' + p + '/*.css'))
-				const clientUpdated = plugins.map(p => 'node_modules/' + p + '/**/*.js');
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(public|static|less)/**/*.less'))
-				const templatesUpdated = plugins.map(p => 'node_modules/' + p + '/**/*.tpl');
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(public|static)/**/*.css'));
-				const langUpdated = plugins.map(p => 'node_modules/' + p + '/**/*.json');
+
+		const styleUpdated_Admin = plugins.map(p => 'node_modules/' + p + '/*.less')
+			.concat(plugins.map(p => 'node_modules/' + p + '/*.css'))
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(public|static|less)/**/*.less'))
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(public|static)/**/*.css'));
+
+		const clientUpdated = plugins.map(p => 'node_modules/' + p + '/+(public|static)/**/*.js');
+		const serverUpdated = plugins.map(p => 'node_modules/' + p + '/*.js')
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(lib|src)/**/*.js'));
+
+		const templatesUpdated = plugins.map(p => 'node_modules/' + p + '/+(public|static|templates)/**/*.tpl');
+		const langUpdated = plugins.map(p => 'node_modules/' + p + '/+(public|static|languages)/**/*.json');
 
 		grunt.config(['watch'], {
-					lessUpdated_Client: {
+			styleUpdated_Client: {
 				files: [
-							'public/less/*.less',
+					'public/less/**/*.less',
-							'!public/less/admin/**/*.less',
+					...styleUpdated_Client,
-							...lessUpdated_Client,
-							'!node_modules/nodebb-*/node_modules/**',
-							'!node_modules/nodebb-*/.git/**',
 				],
 				options: {
 					interval: 1000,
 				},
 			},
-					lessUpdated_Admin: {
+			styleUpdated_Admin: {
 				files: [
-							'public/less/admin/**/*.less',
+					'public/less/**/*.less',
-							...lessUpdated_Admin,
+					...styleUpdated_Admin,
-							'!node_modules/nodebb-*/node_modules/**',
-							'!node_modules/nodebb-*/.git/**',
 				],
 				options: {
 					interval: 1000,
@@ -154,16 +87,23 @@ module.exports = function (grunt) {
 				files: [
 					'public/src/**/*.js',
 					...clientUpdated,
-							'!node_modules/nodebb-*/node_modules/**',
 					'node_modules/benchpressjs/build/benchpress.js',
-							'!node_modules/nodebb-*/.git/**',
 				],
 				options: {
 					interval: 1000,
 				},
 			},
 			serverUpdated: {
-						files: ['*.js', 'install/*.js', 'src/**/*.js', '!src/upgrades/**'],
+				files: [
+					'app.js',
+					'install/*.js',
+					'src/**/*.js',
+					'public/src/modules/translator.js',
+					'public/src/modules/helpers.js',
+					'public/src/utils.js',
+					serverUpdated,
+					'!src/upgrades/**',
+				],
 				options: {
 					interval: 1000,
 				},
@@ -172,8 +112,6 @@ module.exports = function (grunt) {
 				files: [
 					'src/views/**/*.tpl',
 					...templatesUpdated,
-							'!node_modules/nodebb-*/node_modules/**',
-							'!node_modules/nodebb-*/.git/**',
 				],
 				options: {
 					interval: 1000,
@@ -184,49 +122,66 @@ module.exports = function (grunt) {
 					'public/language/en-GB/*.json',
 					'public/language/en-GB/**/*.json',
 					...langUpdated,
-							'!node_modules/nodebb-*/node_modules/**',
-							'!node_modules/nodebb-*/.git/**',
-							'!node_modules/nodebb-*/plugin.json',
-							'!node_modules/nodebb-*/package.json',
-							'!node_modules/nodebb-*/theme.json',
 				],
 				options: {
 					interval: 1000,
 				},
 			},
 		});
-				next();
+		const build = require('./src/meta/build');
-			},
+		if (!grunt.option('skip')) {
-		], done);
+			await build.build(true);
+		}
+		run();
+		done();
 	});
 
-	grunt.task.run('init');
+	function run() {
-
+		if (worker) {
-	env.NODE_ENV = 'development';
+			worker.kill();
-
+		}
-	if (grunt.option('skip')) {
 		worker = fork('app.js', args, {
 			env: env,
 		});
-	} else {
+	}
-		initWorker = fork('app.js', initArgs, {
-			env: env,
-		});
 
-		initWorker.on('exit', function () {
+	grunt.task.run('init');
-			worker = fork('app.js', args, {
+
-				env: env,
+	grunt.event.removeAllListeners('watch');
-			});
+	grunt.event.on('watch', function update(action, filepath, target) {
-		});
+		var compiling;
+		if (target === 'styleUpdated_Client') {
+			compiling = 'clientCSS';
+		} else if (target === 'styleUpdated_Admin') {
+			compiling = 'acpCSS';
+		} else if (target === 'clientUpdated') {
+			compiling = 'js';
+		} else if (target === 'templatesUpdated') {
+			compiling = 'tpl';
+		} else if (target === 'langUpdated') {
+			compiling = 'lang';
+		} else if (target === 'serverUpdated') {
+			// empty require cache
+			const paths = ['./src/meta/build.js', './src/meta/index.js'];
+			paths.forEach(p => delete require.cache[require.resolve(p)]);
+			return run();
 		}
 
-	grunt.event.on('watch', update);
+		require('./src/meta/build').build([compiling], function (err) {
+			if (err) {
+				winston.error(err.stack);
+			}
+			if (worker) {
+				worker.send({ compiling: compiling });
+			}
+		});
+	});
 };
 
-function addBaseThemes(plugins, callback) {
+function addBaseThemes(plugins) {
 	const themeId = plugins.find(p => p.startsWith('nodebb-theme-'));
 	if (!themeId) {
-		return setImmediate(callback, null, plugins);
+		return plugins;
 	}
 	function getBaseRecursive(themeId) {
 		try {
@@ -242,5 +197,5 @@ function addBaseThemes(plugins, callback) {
 	}
 
 	getBaseRecursive(themeId);
-	callback(null, plugins);
+	return plugins;
 }

app.js

@@ -31,6 +31,7 @@ const path = require('path');
 
 const file = require('./src/file');
 
+process.env.NODE_ENV = process.env.NODE_ENV || 'production';
 global.env = process.env.NODE_ENV || 'production';
 
 // Alternate configuration file support

public/src/modules/translator.js

@@ -527,6 +527,12 @@
 		unescape: Translator.unescape,
 		getLanguage: Translator.getLanguage,
 
+		flush: function () {
+			Object.keys(Translator.cache).forEach(function (code) {
+				Translator.cache[code].translations = {};
+			});
+		},
+
 		/**
 		 * Legacy translator function for backwards compatibility
 		 */

src/meta/css.js

@@ -201,7 +201,7 @@ CSS.buildBundle = function (target, fork, callback) {
 			getBundleMetadata(target, next);
 		},
 		function (data, next) {
-			var minify = global.env !== 'development';
+			var minify = process.env.NODE_ENV !== 'development';
 			minifier.css.bundle(data.imports, data.paths, minify, fork, next);
 		},
 		function (bundle, next) {

src/meta/js.js

@@ -260,7 +260,7 @@ JS.buildModules = function (fork, callback) {
 	async.waterfall([
 		clearModules,
 		function (next) {
-			if (global.env === 'development') {
+			if (process.env.NODE_ENV === 'development') {
 				return linkModules(callback);
 			}
 
@@ -323,7 +323,7 @@ function getBundleScriptList(target, callback) {
 
 		var scripts = JS.scripts.base;
 
-		if (target === 'client' && global.env !== 'development') {
+		if (target === 'client' && process.env.NODE_ENV !== 'development') {
 			scripts = scripts.concat(JS.scripts.rjs);
 		} else if (target === 'acp') {
 			scripts = scripts.concat(JS.scripts.admin);
@@ -357,7 +357,7 @@ JS.buildBundle = function (target, fork, callback) {
 			});
 		},
 		function (files, next) {
-			var minify = global.env !== 'development';
+			var minify = process.env.NODE_ENV !== 'development';
 			var filePath = path.join(__dirname, '../../build/public', fileNames[target]);
 
 			minifier.js.bundle({

src/meta/languages.js

@@ -56,7 +56,7 @@ async function getTranslationMetadata() {
 }
 
 async function writeLanguageFile(language, namespace, translations) {
-	const dev = global.env === 'development';
+	const dev = process.env.NODE_ENV === 'development';
 	const filePath = path.join(buildLanguagesPath, language, namespace + '.json');
 
 	await mkdirp(path.dirname(filePath));

src/meta/templates.js

@@ -115,7 +115,7 @@ async function compileTemplate(filename, source) {
 
 	source = await processImports(paths, filename, source);
 	const compiled = await Benchpress.precompile(source, {
-		minify: global.env !== 'development',
+		minify: process.env.NODE_ENV !== 'development',
 	});
 	return await fsWriteFile(path.join(viewsPath, filename.replace(/\.tpl$/, '.js')), compiled);
 }
@@ -139,7 +139,7 @@ async function compile() {
 		await mkdirp(path.join(viewsPath, path.dirname(name)));
 
 		await fsWriteFile(path.join(viewsPath, name), imported);
-		const compiled = await Benchpress.precompile(imported, { minify: global.env !== 'development' });
+		const compiled = await Benchpress.precompile(imported, { minify: process.env.NODE_ENV !== 'development' });
 		await fsWriteFile(path.join(viewsPath, name.replace(/\.tpl$/, '.js')), compiled);
 	}));
 

src/start.js

@@ -118,6 +118,15 @@ function addProcessHandlers() {
 		require('./meta').js.killMinifier();
 		shutdown(1);
 	});
+	process.on('message', function (msg) {
+		if (msg && msg.compiling === 'tpl') {
+			const benchpressjs = require('benchpressjs');
+			benchpressjs.flush();
+		} else if (msg && msg.compiling === 'lang') {
+			const translator = require('./translator');
+			translator.flush();
+		}
+	});
 }
 
 function restart() {