From 2f07c18c5a3fb455f63602108273892e19ce99c0 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 12 Jan 2017 19:31:57 -0700 Subject: [PATCH] All hail the glorious translation prebuilding --- .gitignore | 2 + build.js | 8 +- public/src/modules/translator.js | 5 +- src/admin/search.js | 9 +- src/controllers/index.js | 3 - src/languages.js | 44 +++------- src/meta.js | 1 + src/meta/languages.js | 142 +++++++++++++++++++++++++++++++ src/middleware/index.js | 17 ---- src/middleware/maintenance.js | 2 +- src/plugins.js | 2 - src/plugins/load.js | 60 +------------ src/routes/index.js | 12 ++- 13 files changed, 183 insertions(+), 124 deletions(-) create mode 100644 src/meta/languages.js diff --git a/.gitignore b/.gitignore index 5f402cd667..b5ff54d664 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ tx.exe ##Coverage output coverage + +build diff --git a/build.js b/build.js index a5174d2c70..a0250dfb37 100644 --- a/build.js +++ b/build.js @@ -5,7 +5,7 @@ var winston = require('winston'); var buildStart; -var valid = ['js', 'clientCSS', 'acpCSS', 'tpl']; +var valid = ['js', 'clientCSS', 'acpCSS', 'tpl', 'lang']; exports.buildAll = function (callback) { exports.build(valid.join(','), callback); @@ -88,6 +88,12 @@ exports.buildTargets = function (targets, callback) { startTime = Date.now(); meta.templates.compile(step.bind(this, startTime, target, next)); break; + + case 'lang': + winston.info('[build] Building language files'); + startTime = Date.now(); + meta.languages.build(step.bind(this, startTime, target, next)); + break; default: winston.warn('[build] Unknown build target: \'' + target + '\''); diff --git a/public/src/modules/translator.js b/public/src/modules/translator.js index e4d74b5508..1e71d579e7 100644 --- a/public/src/modules/translator.js +++ b/public/src/modules/translator.js @@ -3,7 +3,7 @@ (function (factory) { 'use strict'; function loadClient(language, namespace) { - return Promise.resolve(jQuery.getJSON(config.relative_path + '/api/language/' + language + '/' + namespace)); + return Promise.resolve(jQuery.getJSON(config.relative_path + '/public/language/' + language + '/' + namespace + '.json')); } var warn = function () {}; if (typeof config === 'object' && config.environment === 'development') { @@ -17,7 +17,6 @@ } else if (typeof module === 'object' && module.exports) { // Node (function () { - require('promise-polyfill'); var languages = require('../../../src/languages'); if (global.env === 'development') { @@ -292,7 +291,7 @@ warn('[translator] Parameter `namespace` is ' + namespace + (namespace === '' ? '(empty string)' : '')); translation = Promise.resolve({}); } else { - translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace); + translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace).catch(function () { return {}; }); } if (key) { diff --git a/src/admin/search.js b/src/admin/search.js index 0bd140e3ba..699f0b33fb 100644 --- a/src/admin/search.js +++ b/src/admin/search.js @@ -5,7 +5,6 @@ var path = require('path'); var async = require('async'); var sanitizeHTML = require('sanitize-html'); -var languages = require('../languages'); var utils = require('../../public/src/utils'); var Translator = require('../../public/src/modules/translator').Translator; @@ -107,6 +106,8 @@ function fallback(namespace, callback) { } function initDict(language, callback) { + var translator = Translator.create(language); + getAdminNamespaces(function (err, namespaces) { if (err) { return callback(err); @@ -115,7 +116,9 @@ function initDict(language, callback) { async.map(namespaces, function (namespace, cb) { async.waterfall([ function (next) { - languages.get(language, namespace, next); + translator.getTranslation(namespace).then(function (translations) { + next(null, translations); + }, next); }, function (translations, next) { if (!translations || !Object.keys(translations).length) { @@ -139,7 +142,7 @@ function initDict(language, callback) { title[1] + '/' + title[2] + ']]') : ''); } - Translator.create(language).translate(title).then(function (title) { + translator.translate(title).then(function (title) { next(null, { namespace: namespace, translations: str + '\n' + title, diff --git a/src/controllers/index.js b/src/controllers/index.js index e6b0d3753c..57943d7078 100644 --- a/src/controllers/index.js +++ b/src/controllers/index.js @@ -351,7 +351,6 @@ Controllers.ping = function (req, res) { Controllers.handle404 = function (req, res) { var relativePath = nconf.get('relative_path'); - var isLanguage = new RegExp('^' + relativePath + '/api/language/.*/.*'); var isClientScript = new RegExp('^' + relativePath + '\\/src\\/.+\\.js'); if (plugins.hasListeners('action:meta.override404')) { @@ -364,8 +363,6 @@ Controllers.handle404 = function (req, res) { if (isClientScript.test(req.url)) { res.type('text/javascript').status(200).send(''); - } else if (isLanguage.test(req.url)) { - res.status(200).json({}); } else if (req.path.startsWith(relativePath + '/uploads') || (req.get('accept') && req.get('accept').indexOf('text/html') === -1) || req.path === '/favicon.ico') { meta.errors.log404(req.path || ''); res.sendStatus(404); diff --git a/src/languages.js b/src/languages.js index f3b9aa5743..5374b9d87f 100644 --- a/src/languages.js +++ b/src/languages.js @@ -3,53 +3,27 @@ var fs = require('fs'); var path = require('path'); var async = require('async'); -var LRU = require('lru-cache'); - -var plugins = require('./plugins'); var Languages = {}; -var languagesPath = path.join(__dirname, '../public/language'); +var languagesPath = path.join(__dirname, '../build/public/language'); Languages.init = function (next) { - if (Languages.hasOwnProperty('_cache')) { - Languages._cache.reset(); - } else { - Languages._cache = LRU(100); - } - next(); }; Languages.get = function (language, namespace, callback) { - var langNamespace = language + '/' + namespace; - - if (Languages._cache && Languages._cache.has(langNamespace)) { - return callback(null, Languages._cache.get(langNamespace)); - } - - var languageData; - fs.readFile(path.join(languagesPath, language, namespace + '.json'), { encoding: 'utf-8' }, function (err, data) { - if (err && err.code !== 'ENOENT') { + if (err) { return callback(err); } - // If language file in core cannot be read, then no language file present try { - languageData = JSON.parse(data) || {}; + data = JSON.parse(data) || {}; } catch (e) { - languageData = {}; + return callback(e); } - if (plugins.customLanguages.hasOwnProperty(langNamespace)) { - Object.assign(languageData, plugins.customLanguages[langNamespace]); - } - - if (Languages._cache) { - Languages._cache.set(langNamespace, languageData); - } - - callback(null, languageData); + callback(null, data); }); }; @@ -73,11 +47,13 @@ Languages.list = function (callback) { var configPath = path.join(languagesPath, folder, 'language.json'); - fs.readFile(configPath, function (err, stream) { - if (err) { + fs.readFile(configPath, function (err, buffer) { + if (err && err.code !== 'ENOENT') { return next(err); } - languages.push(JSON.parse(stream.toString())); + if (buffer) { + languages.push(JSON.parse(buffer.toString())); + } next(); }); }); diff --git a/src/meta.js b/src/meta.js index c732de15f4..2333e54d89 100644 --- a/src/meta.js +++ b/src/meta.js @@ -23,6 +23,7 @@ var utils = require('../public/src/utils'); require('./meta/dependencies')(Meta); Meta.templates = require('./meta/templates'); Meta.blacklist = require('./meta/blacklist'); + Meta.languages = require('./meta/languages'); /* Assorted */ Meta.userOrGroupExists = function (slug, callback) { diff --git a/src/meta/languages.js b/src/meta/languages.js new file mode 100644 index 0000000000..fca4473b0b --- /dev/null +++ b/src/meta/languages.js @@ -0,0 +1,142 @@ +'use strict'; + +var winston = require('winston'); +var path = require('path'); +var async = require('async'); +var fs = require('fs'); +var mkdirp = require('mkdirp'); + +var file = require('../file'); +var utils = require('../../public/src/utils'); +var Plugins = require('../plugins'); +var db = require('../database'); + +var buildLanguagesPath = path.join(__dirname, '../../build/public/language'); +var coreLanguagesPath = path.join(__dirname, '../../public/language'); + +function extrude(languageDir, paths) { + return paths.map(function (p) { + var rel = p.split(languageDir)[1].split(/[\/\\]/).slice(1); + return { + language: rel.shift().replace('_', '-').replace('@', '-x-'), + namespace: rel.join('/').replace(/\.json$/, ''), + path: p, + }; + }); +} + +function getTranslationTree(callback) { + async.waterfall([ + function (next) { + db.getSortedSetRange('plugins:active', 0, -1, next); + }, + function (plugins, next) { + var pluginBasePath = path.join(__dirname, '../../node_modules'); + var paths = plugins.map(function (plugin) { + return path.join(pluginBasePath, plugin); + }); + + // Filter out plugins with invalid paths + async.filter(paths, file.exists, function (paths) { + next(null, paths); + }); + }, + function (paths, next) { + async.map(paths, Plugins.loadPluginInfo, next); + }, + function (plugins, next) { + async.parallel({ + corePaths: function (cb) { + utils.walk(coreLanguagesPath, function (err, paths) { + if (err) { + return cb(err); + } + + cb(null, extrude(coreLanguagesPath, paths)); + }); + }, + pluginPaths: function (nxt) { + plugins = plugins.filter(function (pluginData) { + return (typeof pluginData.languages === 'string'); + }); + async.map(plugins, function (pluginData, cb) { + var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); + utils.walk(pathToFolder, function (err, paths) { + if (err) { + return cb(err); + } + + cb(null, extrude(pathToFolder, paths)); + }); + }, nxt); + } + }, next); + }, + function (data, next) { + var paths = data.pluginPaths.concat.apply([], data.pluginPaths); + paths = data.corePaths.concat(paths); + paths = paths.filter(function (p) { + return p.language && p.namespace && p.path; + }); + + var tree = {}; + + async.eachLimit(paths, 1000, function (data, cb) { + fs.readFile(data.path, function (err, file) { + if (err) { + return cb(err); + } + + try { + var obj = JSON.parse(file.toString()); + + tree[data.language] = tree[data.language] || {}; + tree[data.language][data.namespace] = tree[data.language][data.namespace] || {}; + Object.assign(tree[data.language][data.namespace], obj); + + cb(); + } catch (e) { + winston.warn('[build] Invalid JSON file at `' + data.path + '`'); + cb(); + } + }); + }, function (err) { + next(err, tree); + }); + } + ], callback); +} + +function writeLanguageFiles(tree, callback) { + async.eachLimit(Object.keys(tree), 10, function (language, cb) { + var namespaces = tree[language]; + async.eachLimit(Object.keys(namespaces), 100, function (namespace, next) { + var translations = namespaces[namespace]; + + var filePath = path.join(buildLanguagesPath, language, namespace + '.json'); + + mkdirp(path.dirname(filePath), function (err) { + if (err) { + return next(err); + } + + fs.writeFile(filePath, JSON.stringify(translations), next); + }); + }, cb); + }, callback); +} + +module.exports = { + build: function buildLanguages(callback) { + async.waterfall([ + getTranslationTree, + writeLanguageFiles, + ], function (err) { + if (err) { + winston.error('[build] Language build failed'); + throw err; + } + callback(); + }); + }, +}; diff --git a/src/middleware/index.js b/src/middleware/index.js index f48aaef748..c36f5767ed 100644 --- a/src/middleware/index.js +++ b/src/middleware/index.js @@ -183,23 +183,6 @@ middleware.applyBlacklist = function (req, res, next) { }); }; -middleware.getTranslation = function (req, res, next) { - var language = req.params.language; - var namespace = req.params[0]; - - if (language && namespace) { - languages.get(language, namespace, function (err, translations) { - if (err) { - return next(err); - } - - res.status(200).json(translations); - }); - } else { - res.status(404).json('{}'); - } -}; - middleware.processTimeagoLocales = function (req, res, next) { var fallback = req.path.indexOf('-short') === -1 ? 'jquery.timeago.en.js' : 'jquery.timeago.en-short.js', localPath = path.join(__dirname, '../../public/vendor/jquery/timeago/locales', req.path), diff --git a/src/middleware/maintenance.js b/src/middleware/maintenance.js index 5199fb9332..31ba25e250 100644 --- a/src/middleware/maintenance.js +++ b/src/middleware/maintenance.js @@ -24,7 +24,7 @@ module.exports = function (middleware) { '^/templates/[\\w/]+.tpl', '^/api/login', '^/api/widgets/render', - '^/api/language/.+', + '^/public/language', '^/uploads/system/site-logo.png' ]; diff --git a/src/plugins.js b/src/plugins.js index b8ddbdc0fe..789c92b72c 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -29,8 +29,6 @@ var middleware; Plugins.lessFiles = []; Plugins.clientScripts = []; Plugins.acpScripts = []; - Plugins.customLanguages = {}; - Plugins.customLanguageFallbacks = {}; Plugins.libraryPaths = []; Plugins.versionWarning = []; Plugins.languageCodes = []; diff --git a/src/plugins/load.js b/src/plugins/load.js index 60d584f99f..3121f8457e 100644 --- a/src/plugins/load.js +++ b/src/plugins/load.js @@ -9,8 +9,6 @@ var winston = require('winston'); var nconf = require('nconf'); var _ = require('underscore'); var file = require('../file'); - -var utils = require('../../public/src/utils'); var meta = require('../meta'); @@ -93,9 +91,6 @@ module.exports = function (Plugins) { function (next) { mapClientModules(pluginData, next); }, - function (next) { - loadLanguages(pluginData, next); - } ], function (err) { if (err) { winston.verbose('[plugins] Could not load plugin : ' + pluginData.id); @@ -254,60 +249,6 @@ module.exports = function (Plugins) { callback(); } - function loadLanguages(pluginData, callback) { - if (typeof pluginData.languages !== 'string') { - return callback(); - } - - var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); - var defaultLang = (pluginData.defaultLang || 'en_GB').replace('_', '-').replace('@', '-x-'); - - utils.walk(pathToFolder, function (err, languages) { - if (err) { - return callback(err); - } - - async.each(languages, function (pathToLang, next) { - fs.readFile(pathToLang, function (err, file) { - if (err) { - return next(err); - } - var data; - var language = path.dirname(pathToLang).split(/[\/\\]/).pop().replace('_', '-').replace('@', '-x-'); - var namespace = path.basename(pathToLang, '.json'); - var langNamespace = language + '/' + namespace; - - try { - data = JSON.parse(file.toString()); - } catch (err) { - winston.error('[plugins] Unable to parse custom language file: ' + pathToLang + '\r\n' + err.stack); - return next(err); - } - - Plugins.customLanguages[langNamespace] = Plugins.customLanguages[langNamespace] || {}; - Object.assign(Plugins.customLanguages[langNamespace], data); - - if (defaultLang && defaultLang === language) { - Plugins.languageCodes.filter(function (lang) { - return defaultLang !== lang; - }).forEach(function (lang) { - var langNS = lang + '/' + namespace; - Plugins.customLanguages[langNS] = Object.assign(Plugins.customLanguages[langNS] || {}, data); - }); - } - - next(); - }); - }, function (err) { - if (err) { - return callback(err); - } - - callback(); - }); - }); - } - function resolveModulePath(fullPath, relPath) { /** * With npm@3, dependencies can become flattened, and appear at the root level. @@ -365,6 +306,7 @@ module.exports = function (Plugins) { return callback(new Error('[[error:parse-error]]')); } + callback(null, pluginData); }); }; diff --git a/src/routes/index.js b/src/routes/index.js index 863715008e..ea86434570 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -144,7 +144,17 @@ module.exports = function (app, middleware, hotswapIds) { } app.use(middleware.privateUploads); - app.use(relativePath + '/api/language/:language/(([a-zA-Z0-9\\-_.\\/]+))', middleware.getTranslation); + app.use(relativePath + '/public/language', express.static(path.join(__dirname, '../../', 'build/public/language'), { + maxAge: app.enabled('cache') ? 5184000000 : 0 + })); + + // DEPRECATED + app.use(relativePath + '/api/language', function (req, res) { + winston.warn('[deprecated] Accessing language files from `/api/language` is deprecated. ' + + 'Use `/public/language/[langCode]/[namespace].json` for prefetch paths.'); + res.redirect(relativePath + '/public/language' + req.path + '.json'); + }); + app.use(relativePath, express.static(path.join(__dirname, '../../', 'public'), { maxAge: app.enabled('cache') ? 5184000000 : 0 }));