From cb662e15ce673e1feafda9a41adcfcec0eb9ce97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?=
 <barisusakli@gmail.com>
Date: Tue, 19 May 2020 21:15:51 -0400
Subject: [PATCH] feat: improve grunt restart/rebuild speed

---
 Gruntfile.js                     | 337 +++++++++++++------------------
 app.js                           |   1 +
 public/src/modules/translator.js |   6 +
 src/meta/css.js                  |   2 +-
 src/meta/js.js                   |   6 +-
 src/meta/languages.js            |   2 +-
 src/meta/templates.js            |   4 +-
 src/start.js                     |   9 +
 8 files changed, 169 insertions(+), 198 deletions(-)

diff --git a/Gruntfile.js b/Gruntfile.js
index a08ac32bae..b0a6abd548 100644
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -1,95 +1,31 @@
 'use strict';
 
-
-var async = require('async');
-var fork = require('child_process').fork;
-var env = process.env;
+const path = require('path');
+const nconf = require('nconf');
+nconf.argv().env({
+	separator: '__',
+});
+const winston = require('winston');
+const fork = require('child_process').fork;
+const env = process.env;
 var worker;
-var updateWorker;
-var initWorker;
-var incomplete = [];
-var running = 0;
 
 env.NODE_ENV = env.NODE_ENV || 'development';
 
+const configFile = path.resolve(__dirname, nconf.any(['config', 'CONFIG']) || 'config.json');
+const prestart = require('./src/prestart');
+prestart.loadConfig(configFile);
 
-var nconf = require('nconf');
-nconf.file({
-	file: 'config.json',
-});
-
-nconf.defaults({
-	base_dir: __dirname,
-	views_dir: './build/public/templates',
-});
-var winston = require('winston');
-winston.configure({
-	transports: [
-		new winston.transports.Console({
-			handleExceptions: true,
-		}),
-	],
-});
 var db = require('./src/database');
 
 module.exports = function (grunt) {
 	var args = [];
-	var initArgs = ['--build'];
+
 	if (!grunt.option('verbose')) {
 		args.push('--log-level=info');
-		initArgs.push('--log-level=info');
-	}
-
-	function update(action, filepath, target) {
-		var updateArgs = args.slice();
-		var compiling;
-		var time = Date.now();
-
-		if (target === 'lessUpdated_Client') {
-			compiling = 'clientCSS';
-		} else if (target === 'lessUpdated_Admin') {
-			compiling = 'acpCSS';
-		} else if (target === 'clientUpdated') {
-			compiling = 'js';
-		} else if (target === 'templatesUpdated') {
-			compiling = 'tpl';
-		} else if (target === 'langUpdated') {
-			compiling = 'lang';
-		} else if (target === 'serverUpdated') {
-			// Do nothing, just restart
-		}
-
-		if (compiling && !incomplete.includes(compiling)) {
-			incomplete.push(compiling);
-		}
-
-		updateArgs.push('--build');
-		updateArgs.push(incomplete.join(','));
-
-		worker.kill();
-		if (updateWorker) {
-			updateWorker.kill('SIGKILL');
-		}
-		updateWorker = fork('app.js', updateArgs, { env: env });
-		running += 1;
-		updateWorker.on('exit', function () {
-			running -= 1;
-			if (running === 0) {
-				worker = fork('app.js', args, {
-					env: env,
-				});
-				worker.on('message', function () {
-					if (incomplete.length) {
-						incomplete = [];
-
-						if (grunt.option('verbose')) {
-							grunt.log.writeln('NodeBB restarted in ' + (Date.now() - time) + ' ms');
-						}
-					}
-				});
-			}
-		});
+		nconf.set('log-level', 'info');
 	}
+	prestart.setupWinston();
 
 	grunt.initConfig({
 		watch: {},
@@ -99,134 +35,153 @@ module.exports = function (grunt) {
 
 	grunt.registerTask('default', ['watch']);
 
-	grunt.registerTask('init', function () {
+	grunt.registerTask('init', async function () {
 		var done = this.async();
-		async.waterfall([
-			function (next) {
-				db.init(next);
+		let plugins = [];
+		if (!process.argv.includes('--core')) {
+			await db.init();
+			plugins = await db.getSortedSetRange('plugins:active', 0, -1);
+			addBaseThemes(plugins);
+			if (!plugins.includes('nodebb-plugin-composer-default')) {
+				plugins.push('nodebb-plugin-composer-default');
+			}
+		}
+
+		const styleUpdated_Client = plugins.map(p => 'node_modules/' + p + '/*.less')
+			.concat(plugins.map(p => 'node_modules/' + p + '/*.css'))
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(public|static|less)/**/*.less'))
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(public|static)/**/*.css'));
+
+		const styleUpdated_Admin = plugins.map(p => 'node_modules/' + p + '/*.less')
+			.concat(plugins.map(p => 'node_modules/' + p + '/*.css'))
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(public|static|less)/**/*.less'))
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(public|static)/**/*.css'));
+
+		const clientUpdated = plugins.map(p => 'node_modules/' + p + '/+(public|static)/**/*.js');
+		const serverUpdated = plugins.map(p => 'node_modules/' + p + '/*.js')
+			.concat(plugins.map(p => 'node_modules/' + p + '/+(lib|src)/**/*.js'));
+
+		const templatesUpdated = plugins.map(p => 'node_modules/' + p + '/+(public|static|templates)/**/*.tpl');
+		const langUpdated = plugins.map(p => 'node_modules/' + p + '/+(public|static|languages)/**/*.json');
+
+		grunt.config(['watch'], {
+			styleUpdated_Client: {
+				files: [
+					'public/less/**/*.less',
+					...styleUpdated_Client,
+				],
+				options: {
+					interval: 1000,
+				},
 			},
-			function (next) {
-				db.getSortedSetRange('plugins:active', 0, -1, next);
+			styleUpdated_Admin: {
+				files: [
+					'public/less/**/*.less',
+					...styleUpdated_Admin,
+				],
+				options: {
+					interval: 1000,
+				},
 			},
-			function (plugins, next) {
-				addBaseThemes(plugins, next);
+			clientUpdated: {
+				files: [
+					'public/src/**/*.js',
+					...clientUpdated,
+					'node_modules/benchpressjs/build/benchpress.js',
+				],
+				options: {
+					interval: 1000,
+				},
 			},
-			function (plugins, next) {
-				if (!plugins.includes('nodebb-plugin-composer-default')) {
-					plugins.push('nodebb-plugin-composer-default');
-				}
-
-				if (process.argv.includes('--core')) {
-					plugins = [];
-				}
-
-				const lessUpdated_Client = plugins.map(p => 'node_modules/' + p + '/**/*.less');
-				const lessUpdated_Admin = plugins.map(p => 'node_modules/' + p + '/**/*.less');
-				const clientUpdated = plugins.map(p => 'node_modules/' + p + '/**/*.js');
-				const templatesUpdated = plugins.map(p => 'node_modules/' + p + '/**/*.tpl');
-				const langUpdated = plugins.map(p => 'node_modules/' + p + '/**/*.json');
-
-				grunt.config(['watch'], {
-					lessUpdated_Client: {
-						files: [
-							'public/less/*.less',
-							'!public/less/admin/**/*.less',
-							...lessUpdated_Client,
-							'!node_modules/nodebb-*/node_modules/**',
-							'!node_modules/nodebb-*/.git/**',
-						],
-						options: {
-							interval: 1000,
-						},
-					},
-					lessUpdated_Admin: {
-						files: [
-							'public/less/admin/**/*.less',
-							...lessUpdated_Admin,
-							'!node_modules/nodebb-*/node_modules/**',
-							'!node_modules/nodebb-*/.git/**',
-						],
-						options: {
-							interval: 1000,
-						},
-					},
-					clientUpdated: {
-						files: [
-							'public/src/**/*.js',
-							...clientUpdated,
-							'!node_modules/nodebb-*/node_modules/**',
-							'node_modules/benchpressjs/build/benchpress.js',
-							'!node_modules/nodebb-*/.git/**',
-						],
-						options: {
-							interval: 1000,
-						},
-					},
-					serverUpdated: {
-						files: ['*.js', 'install/*.js', 'src/**/*.js', '!src/upgrades/**'],
-						options: {
-							interval: 1000,
-						},
-					},
-					templatesUpdated: {
-						files: [
-							'src/views/**/*.tpl',
-							...templatesUpdated,
-							'!node_modules/nodebb-*/node_modules/**',
-							'!node_modules/nodebb-*/.git/**',
-						],
-						options: {
-							interval: 1000,
-						},
-					},
-					langUpdated: {
-						files: [
-							'public/language/en-GB/*.json',
-							'public/language/en-GB/**/*.json',
-							...langUpdated,
-							'!node_modules/nodebb-*/node_modules/**',
-							'!node_modules/nodebb-*/.git/**',
-							'!node_modules/nodebb-*/plugin.json',
-							'!node_modules/nodebb-*/package.json',
-							'!node_modules/nodebb-*/theme.json',
-						],
-						options: {
-							interval: 1000,
-						},
-					},
-				});
-				next();
+			serverUpdated: {
+				files: [
+					'app.js',
+					'install/*.js',
+					'src/**/*.js',
+					'public/src/modules/translator.js',
+					'public/src/modules/helpers.js',
+					'public/src/utils.js',
+					serverUpdated,
+					'!src/upgrades/**',
+				],
+				options: {
+					interval: 1000,
+				},
 			},
-		], done);
+			templatesUpdated: {
+				files: [
+					'src/views/**/*.tpl',
+					...templatesUpdated,
+				],
+				options: {
+					interval: 1000,
+				},
+			},
+			langUpdated: {
+				files: [
+					'public/language/en-GB/*.json',
+					'public/language/en-GB/**/*.json',
+					...langUpdated,
+				],
+				options: {
+					interval: 1000,
+				},
+			},
+		});
+		const build = require('./src/meta/build');
+		if (!grunt.option('skip')) {
+			await build.build(true);
+		}
+		run();
+		done();
 	});
 
-	grunt.task.run('init');
-
-	env.NODE_ENV = 'development';
-
-	if (grunt.option('skip')) {
+	function run() {
+		if (worker) {
+			worker.kill();
+		}
 		worker = fork('app.js', args, {
 			env: env,
 		});
-	} else {
-		initWorker = fork('app.js', initArgs, {
-			env: env,
-		});
-
-		initWorker.on('exit', function () {
-			worker = fork('app.js', args, {
-				env: env,
-			});
-		});
 	}
 
-	grunt.event.on('watch', update);
+	grunt.task.run('init');
+
+	grunt.event.removeAllListeners('watch');
+	grunt.event.on('watch', function update(action, filepath, target) {
+		var compiling;
+		if (target === 'styleUpdated_Client') {
+			compiling = 'clientCSS';
+		} else if (target === 'styleUpdated_Admin') {
+			compiling = 'acpCSS';
+		} else if (target === 'clientUpdated') {
+			compiling = 'js';
+		} else if (target === 'templatesUpdated') {
+			compiling = 'tpl';
+		} else if (target === 'langUpdated') {
+			compiling = 'lang';
+		} else if (target === 'serverUpdated') {
+			// empty require cache
+			const paths = ['./src/meta/build.js', './src/meta/index.js'];
+			paths.forEach(p => delete require.cache[require.resolve(p)]);
+			return run();
+		}
+
+		require('./src/meta/build').build([compiling], function (err) {
+			if (err) {
+				winston.error(err.stack);
+			}
+			if (worker) {
+				worker.send({ compiling: compiling });
+			}
+		});
+	});
 };
 
-function addBaseThemes(plugins, callback) {
+function addBaseThemes(plugins) {
 	const themeId = plugins.find(p => p.startsWith('nodebb-theme-'));
 	if (!themeId) {
-		return setImmediate(callback, null, plugins);
+		return plugins;
 	}
 	function getBaseRecursive(themeId) {
 		try {
@@ -242,5 +197,5 @@ function addBaseThemes(plugins, callback) {
 	}
 
 	getBaseRecursive(themeId);
-	callback(null, plugins);
+	return plugins;
 }
diff --git a/app.js b/app.js
index 3c83082bb8..cf4d0d97f2 100644
--- a/app.js
+++ b/app.js
@@ -31,6 +31,7 @@ const path = require('path');
 
 const file = require('./src/file');
 
+process.env.NODE_ENV = process.env.NODE_ENV || 'production';
 global.env = process.env.NODE_ENV || 'production';
 
 // Alternate configuration file support
diff --git a/public/src/modules/translator.js b/public/src/modules/translator.js
index 5c7b3366e2..aed6760675 100644
--- a/public/src/modules/translator.js
+++ b/public/src/modules/translator.js
@@ -527,6 +527,12 @@
 		unescape: Translator.unescape,
 		getLanguage: Translator.getLanguage,
 
+		flush: function () {
+			Object.keys(Translator.cache).forEach(function (code) {
+				Translator.cache[code].translations = {};
+			});
+		},
+
 		/**
 		 * Legacy translator function for backwards compatibility
 		 */
diff --git a/src/meta/css.js b/src/meta/css.js
index b8cb9a1a8f..3af72f0971 100644
--- a/src/meta/css.js
+++ b/src/meta/css.js
@@ -201,7 +201,7 @@ CSS.buildBundle = function (target, fork, callback) {
 			getBundleMetadata(target, next);
 		},
 		function (data, next) {
-			var minify = global.env !== 'development';
+			var minify = process.env.NODE_ENV !== 'development';
 			minifier.css.bundle(data.imports, data.paths, minify, fork, next);
 		},
 		function (bundle, next) {
diff --git a/src/meta/js.js b/src/meta/js.js
index aa329022d9..4a9ef52ccf 100644
--- a/src/meta/js.js
+++ b/src/meta/js.js
@@ -260,7 +260,7 @@ JS.buildModules = function (fork, callback) {
 	async.waterfall([
 		clearModules,
 		function (next) {
-			if (global.env === 'development') {
+			if (process.env.NODE_ENV === 'development') {
 				return linkModules(callback);
 			}
 
@@ -323,7 +323,7 @@ function getBundleScriptList(target, callback) {
 
 		var scripts = JS.scripts.base;
 
-		if (target === 'client' && global.env !== 'development') {
+		if (target === 'client' && process.env.NODE_ENV !== 'development') {
 			scripts = scripts.concat(JS.scripts.rjs);
 		} else if (target === 'acp') {
 			scripts = scripts.concat(JS.scripts.admin);
@@ -357,7 +357,7 @@ JS.buildBundle = function (target, fork, callback) {
 			});
 		},
 		function (files, next) {
-			var minify = global.env !== 'development';
+			var minify = process.env.NODE_ENV !== 'development';
 			var filePath = path.join(__dirname, '../../build/public', fileNames[target]);
 
 			minifier.js.bundle({
diff --git a/src/meta/languages.js b/src/meta/languages.js
index 7f4afd0a02..63ec3c163d 100644
--- a/src/meta/languages.js
+++ b/src/meta/languages.js
@@ -56,7 +56,7 @@ async function getTranslationMetadata() {
 }
 
 async function writeLanguageFile(language, namespace, translations) {
-	const dev = global.env === 'development';
+	const dev = process.env.NODE_ENV === 'development';
 	const filePath = path.join(buildLanguagesPath, language, namespace + '.json');
 
 	await mkdirp(path.dirname(filePath));
diff --git a/src/meta/templates.js b/src/meta/templates.js
index e3f5005d78..eb5f2f5d8d 100644
--- a/src/meta/templates.js
+++ b/src/meta/templates.js
@@ -115,7 +115,7 @@ async function compileTemplate(filename, source) {
 
 	source = await processImports(paths, filename, source);
 	const compiled = await Benchpress.precompile(source, {
-		minify: global.env !== 'development',
+		minify: process.env.NODE_ENV !== 'development',
 	});
 	return await fsWriteFile(path.join(viewsPath, filename.replace(/\.tpl$/, '.js')), compiled);
 }
@@ -139,7 +139,7 @@ async function compile() {
 		await mkdirp(path.join(viewsPath, path.dirname(name)));
 
 		await fsWriteFile(path.join(viewsPath, name), imported);
-		const compiled = await Benchpress.precompile(imported, { minify: global.env !== 'development' });
+		const compiled = await Benchpress.precompile(imported, { minify: process.env.NODE_ENV !== 'development' });
 		await fsWriteFile(path.join(viewsPath, name.replace(/\.tpl$/, '.js')), compiled);
 	}));
 
diff --git a/src/start.js b/src/start.js
index 442bba134e..f103aed7be 100644
--- a/src/start.js
+++ b/src/start.js
@@ -118,6 +118,15 @@ function addProcessHandlers() {
 		require('./meta').js.killMinifier();
 		shutdown(1);
 	});
+	process.on('message', function (msg) {
+		if (msg && msg.compiling === 'tpl') {
+			const benchpressjs = require('benchpressjs');
+			benchpressjs.flush();
+		} else if (msg && msg.compiling === 'lang') {
+			const translator = require('./translator');
+			translator.flush();
+		}
+	});
 }
 
 function restart() {