All hail the glorious translation prebuilding

.gitignore

@@ -55,3 +55,5 @@ tx.exe
 
 ##Coverage output
 coverage
+
+build

build.js

@@ -5,7 +5,7 @@ var winston = require('winston');
 
 var buildStart;
 
-var valid = ['js', 'clientCSS', 'acpCSS', 'tpl'];
+var valid = ['js', 'clientCSS', 'acpCSS', 'tpl', 'lang'];
 
 exports.buildAll = function (callback) {
 	exports.build(valid.join(','), callback);
@@ -89,6 +89,12 @@ exports.buildTargets = function (targets, callback) {
 						meta.templates.compile(step.bind(this, startTime, target, next));
 						break;
 					
+					case 'lang':
+						winston.info('[build] Building language files');
+						startTime = Date.now();
+						meta.languages.build(step.bind(this, startTime, target, next));
+						break;
+
 					default:
 						winston.warn('[build] Unknown build target: \'' + target + '\'');
 						setImmediate(next);

public/src/modules/translator.js

@@ -3,7 +3,7 @@
 (function (factory) {
 	'use strict';
 	function loadClient(language, namespace) {
-		return Promise.resolve(jQuery.getJSON(config.relative_path + '/api/language/' + language + '/' + namespace));
+		return Promise.resolve(jQuery.getJSON(config.relative_path + '/public/language/' + language + '/' + namespace + '.json'));
 	}
 	var warn = function () {};
 	if (typeof config === 'object' && config.environment === 'development') {
@@ -17,7 +17,6 @@
 	} else if (typeof module === 'object' && module.exports) {
 		// Node
 		(function () {
-			require('promise-polyfill');
 			var languages = require('../../../src/languages');
 
 			if (global.env === 'development') {
@@ -292,7 +291,7 @@
 				warn('[translator] Parameter `namespace` is ' + namespace + (namespace === '' ? '(empty string)' : ''));
 				translation = Promise.resolve({});
 			} else {
-				translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace);
+				translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace).catch(function () { return {}; });
 			}
 
 			if (key) {

src/admin/search.js

@@ -5,7 +5,6 @@ var path = require('path');
 var async = require('async');
 var sanitizeHTML = require('sanitize-html');
 
-var languages = require('../languages');
 var utils = require('../../public/src/utils');
 var Translator = require('../../public/src/modules/translator').Translator;
 
@@ -107,6 +106,8 @@ function fallback(namespace, callback) {
 }
 
 function initDict(language, callback) {
+	var translator = Translator.create(language);
+
 	getAdminNamespaces(function (err, namespaces) {
 		if (err) {
 			return callback(err);
@@ -115,7 +116,9 @@ function initDict(language, callback) {
 		async.map(namespaces, function (namespace, cb) {
 			async.waterfall([
 				function (next) {
-					languages.get(language, namespace, next);
+					translator.getTranslation(namespace).then(function (translations) {
+						next(null, translations);
+					}, next);
 				},
 				function (translations, next) {
 					if (!translations || !Object.keys(translations).length) {
@@ -139,7 +142,7 @@ function initDict(language, callback) {
 							title[1] + '/' + title[2] + ']]') : '');
 					}
 
-					Translator.create(language).translate(title).then(function (title) {
+					translator.translate(title).then(function (title) {
 						next(null, {
 							namespace: namespace,
 							translations: str + '\n' + title,

src/controllers/index.js

@@ -351,7 +351,6 @@ Controllers.ping = function (req, res) {
 
 Controllers.handle404 = function (req, res) {
 	var relativePath = nconf.get('relative_path');
-	var isLanguage = new RegExp('^' + relativePath + '/api/language/.*/.*');
 	var isClientScript = new RegExp('^' + relativePath + '\\/src\\/.+\\.js');
 
 	if (plugins.hasListeners('action:meta.override404')) {
@@ -364,8 +363,6 @@ Controllers.handle404 = function (req, res) {
 
 	if (isClientScript.test(req.url)) {
 		res.type('text/javascript').status(200).send('');
-	} else if (isLanguage.test(req.url)) {
-		res.status(200).json({});
 	} else if (req.path.startsWith(relativePath + '/uploads') || (req.get('accept') && req.get('accept').indexOf('text/html') === -1) || req.path === '/favicon.ico') {
 		meta.errors.log404(req.path || '');
 		res.sendStatus(404);

src/languages.js

@@ -3,53 +3,27 @@
 var fs = require('fs');
 var path = require('path');
 var async = require('async');
-var LRU = require('lru-cache');
-
-var plugins = require('./plugins');
 
 var Languages = {};
-var	languagesPath = path.join(__dirname, '../public/language');
+var	languagesPath = path.join(__dirname, '../build/public/language');
 
 Languages.init = function (next) {
-	if (Languages.hasOwnProperty('_cache')) {
-		Languages._cache.reset();
-	} else {
-		Languages._cache = LRU(100);
-	}
-
 	next();
 };
 
 Languages.get = function (language, namespace, callback) {
-	var langNamespace = language + '/' + namespace;
-
-	if (Languages._cache && Languages._cache.has(langNamespace)) {
-		return callback(null, Languages._cache.get(langNamespace));
-	}
-
-	var languageData;
-
 	fs.readFile(path.join(languagesPath, language, namespace + '.json'), { encoding: 'utf-8' }, function (err, data) {
-		if (err && err.code !== 'ENOENT') {
+		if (err) {
 			return callback(err);
 		}
 
-		// If language file in core cannot be read, then no language file present
 		try {
-			languageData = JSON.parse(data) || {};
+			data = JSON.parse(data) || {};
 		} catch (e) {
-			languageData = {};
+			return callback(e);
 		}
 
-		if (plugins.customLanguages.hasOwnProperty(langNamespace)) {
+		callback(null, data);
-			Object.assign(languageData, plugins.customLanguages[langNamespace]);
-		}
-
-		if (Languages._cache) {
-			Languages._cache.set(langNamespace, languageData);
-		}
-
-		callback(null, languageData);
 	});
 };
 
@@ -73,11 +47,13 @@ Languages.list = function (callback) {
 
 				var configPath = path.join(languagesPath, folder, 'language.json');
 
-				fs.readFile(configPath, function (err, stream) {
+				fs.readFile(configPath, function (err, buffer) {
-					if (err) {
+					if (err && err.code !== 'ENOENT') {
 						return next(err);
 					}
-					languages.push(JSON.parse(stream.toString()));
+					if (buffer) {
+						languages.push(JSON.parse(buffer.toString()));
+					}
 					next();
 				});
 			});

src/meta.js

@@ -23,6 +23,7 @@ var utils = require('../public/src/utils');
 	require('./meta/dependencies')(Meta);
 	Meta.templates = require('./meta/templates');
 	Meta.blacklist = require('./meta/blacklist');
+	Meta.languages = require('./meta/languages');
 
 	/* Assorted */
 	Meta.userOrGroupExists = function (slug, callback) {

src/meta/languages.js

@@ -0,0 +1,142 @@
+'use strict';
+
+var winston = require('winston');
+var path = require('path');
+var async = require('async');
+var fs = require('fs');
+var mkdirp = require('mkdirp');
+
+var file = require('../file');
+var utils = require('../../public/src/utils');
+var Plugins = require('../plugins');
+var db = require('../database');
+
+var buildLanguagesPath = path.join(__dirname, '../../build/public/language');
+var	coreLanguagesPath = path.join(__dirname, '../../public/language');
+
+function extrude(languageDir, paths) {
+	return paths.map(function (p) {
+		var rel = p.split(languageDir)[1].split(/[\/\\]/).slice(1);
+		return {
+			language: rel.shift().replace('_', '-').replace('@', '-x-'),
+			namespace: rel.join('/').replace(/\.json$/, ''),
+			path: p,
+		};
+	});
+}
+
+function getTranslationTree(callback) {
+	async.waterfall([
+		function (next) {
+			db.getSortedSetRange('plugins:active', 0, -1, next);
+		},
+		function (plugins, next) {
+			var pluginBasePath = path.join(__dirname, '../../node_modules');
+			var paths = plugins.map(function (plugin) {
+				return path.join(pluginBasePath, plugin);
+			});
+
+			// Filter out plugins with invalid paths
+			async.filter(paths, file.exists, function (paths) {
+				next(null, paths);
+			});
+		},
+		function (paths, next) {
+			async.map(paths, Plugins.loadPluginInfo, next);
+		},
+		function (plugins, next) {
+			async.parallel({
+				corePaths: function (cb) {
+					utils.walk(coreLanguagesPath, function (err, paths) {
+						if (err) {
+							return cb(err);
+						}
+
+						cb(null, extrude(coreLanguagesPath, paths));
+					});
+				},
+				pluginPaths: function (nxt) {
+					plugins = plugins.filter(function (pluginData) {
+						return (typeof pluginData.languages === 'string');
+					});
+					async.map(plugins, function (pluginData, cb) {
+						var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages);
+						utils.walk(pathToFolder, function (err, paths) {
+							if (err) {
+								return cb(err);
+							}
+
+							cb(null, extrude(pathToFolder, paths));
+						});
+					}, nxt);
+				}
+			}, next);
+		},
+		function (data, next) {
+			var paths = data.pluginPaths.concat.apply([], data.pluginPaths);
+			paths = data.corePaths.concat(paths);
+			paths = paths.filter(function (p) {
+				return p.language && p.namespace && p.path;
+			});
+
+			var tree = {};
+			
+			async.eachLimit(paths, 1000, function (data, cb) {
+				fs.readFile(data.path, function (err, file) {
+					if (err) {
+						return cb(err);
+					}
+
+					try {
+						var obj = JSON.parse(file.toString());
+
+						tree[data.language] = tree[data.language] || {};
+						tree[data.language][data.namespace] = tree[data.language][data.namespace] || {};
+						Object.assign(tree[data.language][data.namespace], obj);
+						
+						cb();
+					} catch (e) {
+						winston.warn('[build] Invalid JSON file at `' + data.path + '`');
+						cb();
+					}
+				});
+			}, function (err) {
+				next(err, tree);
+			});
+		}
+	], callback);
+}
+
+function writeLanguageFiles(tree, callback) {
+	async.eachLimit(Object.keys(tree), 10, function (language, cb) {
+		var namespaces = tree[language];
+		async.eachLimit(Object.keys(namespaces), 100, function (namespace, next) {
+			var translations = namespaces[namespace];
+
+			var filePath = path.join(buildLanguagesPath, language, namespace + '.json');
+
+			mkdirp(path.dirname(filePath), function (err) {
+				if (err) {
+					return next(err);
+				}
+
+				fs.writeFile(filePath, JSON.stringify(translations), next);
+			});
+		}, cb);
+	}, callback);
+}
+
+module.exports = {
+	build: function buildLanguages(callback) {
+		async.waterfall([
+			getTranslationTree,
+			writeLanguageFiles,
+		], function (err) {
+			if (err) {
+				winston.error('[build] Language build failed');
+				throw err;
+			}
+			callback();
+		});
+	},
+};

src/middleware/index.js

@@ -183,23 +183,6 @@ middleware.applyBlacklist = function (req, res, next) {
 	});
 };
 
-middleware.getTranslation = function (req, res, next) {
-	var language = req.params.language;
-	var namespace = req.params[0];
-
-	if (language && namespace) {
-		languages.get(language, namespace, function (err, translations) {
-			if (err) {
-				return next(err);
-			}
-
-			res.status(200).json(translations);
-		});
-	} else {
-		res.status(404).json('{}');
-	}
-};
-
 middleware.processTimeagoLocales = function (req, res, next) {
 	var fallback = req.path.indexOf('-short') === -1 ? 'jquery.timeago.en.js' : 'jquery.timeago.en-short.js',
 		localPath = path.join(__dirname, '../../public/vendor/jquery/timeago/locales', req.path),

src/middleware/maintenance.js

@@ -24,7 +24,7 @@ module.exports = function (middleware) {
 			'^/templates/[\\w/]+.tpl',
 			'^/api/login',
 			'^/api/widgets/render',
-			'^/api/language/.+',
+			'^/public/language',
 			'^/uploads/system/site-logo.png'
 		];
 

src/plugins.js

@@ -29,8 +29,6 @@ var middleware;
 	Plugins.lessFiles = [];
 	Plugins.clientScripts = [];
 	Plugins.acpScripts = [];
-	Plugins.customLanguages = {};
-	Plugins.customLanguageFallbacks = {};
 	Plugins.libraryPaths = [];
 	Plugins.versionWarning = [];
 	Plugins.languageCodes = [];

src/plugins/load.js

@@ -9,8 +9,6 @@ var winston = require('winston');
 var nconf = require('nconf');
 var _ = require('underscore');
 var file = require('../file');
-
-var utils = require('../../public/src/utils');
 var meta = require('../meta');
 
 
@@ -93,9 +91,6 @@ module.exports = function (Plugins) {
 				function (next) {
 					mapClientModules(pluginData, next);
 				},
-				function (next) {
-					loadLanguages(pluginData, next);
-				}
 			], function (err) {
 				if (err) {
 					winston.verbose('[plugins] Could not load plugin : ' + pluginData.id);
@@ -254,60 +249,6 @@ module.exports = function (Plugins) {
 		callback();
 	}
 
-	function loadLanguages(pluginData, callback) {
-		if (typeof pluginData.languages !== 'string') {
-			return callback();
-		}
-
-		var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages);
-		var defaultLang = (pluginData.defaultLang || 'en_GB').replace('_', '-').replace('@', '-x-');
-
-		utils.walk(pathToFolder, function (err, languages) {
-			if (err) {
-				return callback(err);
-			}
-
-			async.each(languages, function (pathToLang, next) {
-				fs.readFile(pathToLang, function (err, file) {
-					if (err) {
-						return next(err);
-					}
-					var data;
-					var language = path.dirname(pathToLang).split(/[\/\\]/).pop().replace('_', '-').replace('@', '-x-');
-					var namespace = path.basename(pathToLang, '.json');
-					var langNamespace = language + '/' + namespace;
-
-					try {
-						data = JSON.parse(file.toString());
-					} catch (err) {
-						winston.error('[plugins] Unable to parse custom language file: ' + pathToLang + '\r\n' + err.stack);
-						return next(err);
-					}
-
-					Plugins.customLanguages[langNamespace] = Plugins.customLanguages[langNamespace] || {};
-					Object.assign(Plugins.customLanguages[langNamespace], data);
-
-					if (defaultLang && defaultLang === language) {
-						Plugins.languageCodes.filter(function (lang) {
-							return defaultLang !== lang;
-						}).forEach(function (lang) {
-							var langNS = lang + '/' + namespace;
-							Plugins.customLanguages[langNS] = Object.assign(Plugins.customLanguages[langNS] || {}, data);
-						});
-					}
-
-					next();
-				});
-			}, function (err) {
-				if (err) {
-					return callback(err);
-				}
-
-				callback();
-			});
-		});
-	}
-
 	function resolveModulePath(fullPath, relPath) {
 		/**
 		  * With npm@3, dependencies can become flattened, and appear at the root level.
@@ -365,6 +306,7 @@ module.exports = function (Plugins) {
 
 				return callback(new Error('[[error:parse-error]]'));
 			}
+
 			callback(null, pluginData);
 		});
 	};

src/routes/index.js

@@ -144,7 +144,17 @@ module.exports = function (app, middleware, hotswapIds) {
 	}
 
 	app.use(middleware.privateUploads);
-	app.use(relativePath + '/api/language/:language/(([a-zA-Z0-9\\-_.\\/]+))', middleware.getTranslation);
+	app.use(relativePath + '/public/language', express.static(path.join(__dirname, '../../', 'build/public/language'), {
+		maxAge: app.enabled('cache') ? 5184000000 : 0
+	}));
+
+	// DEPRECATED
+	app.use(relativePath + '/api/language', function (req, res) {
+		winston.warn('[deprecated] Accessing language files from `/api/language` is deprecated. ' + 
+			'Use `/public/language/[langCode]/[namespace].json` for prefetch paths.');
+		res.redirect(relativePath + '/public/language' + req.path + '.json');
+	});
+
 	app.use(relativePath, express.static(path.join(__dirname, '../../', 'public'), {
 		maxAge: app.enabled('cache') ? 5184000000 : 0
 	}));