From 386284beb83ad76ec1e44cd9c88d5107b00a10c8 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 12 Jan 2017 15:45:44 -0700 Subject: [PATCH 01/10] Windows compat --- nodebb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebb b/nodebb index 62115e110b..e1a4ab7fbf 100755 --- a/nodebb +++ b/nodebb @@ -375,7 +375,7 @@ switch(process.argv[2]) { async.series([ function (next) { process.stdout.write('1. '.bold + 'Bringing base dependencies up to date... '.yellow); - require('child_process').execFile('/usr/bin/env', ['npm', 'i', '--production'], { stdio: 'ignore' }, next); + cproc.exec('npm i --production', { cwd: __dirname, stdio: 'ignore' }, next); }, function (next) { process.stdout.write('OK\n'.green); From 2f07c18c5a3fb455f63602108273892e19ce99c0 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 12 Jan 2017 19:31:57 -0700 Subject: [PATCH 02/10] All hail the glorious translation prebuilding --- .gitignore | 2 + build.js | 8 +- public/src/modules/translator.js | 5 +- src/admin/search.js | 9 +- src/controllers/index.js | 3 - src/languages.js | 44 +++------- src/meta.js | 1 + src/meta/languages.js | 142 +++++++++++++++++++++++++++++++ src/middleware/index.js | 17 ---- src/middleware/maintenance.js | 2 +- src/plugins.js | 2 - src/plugins/load.js | 60 +------------ src/routes/index.js | 12 ++- 13 files changed, 183 insertions(+), 124 deletions(-) create mode 100644 src/meta/languages.js diff --git a/.gitignore b/.gitignore index 5f402cd667..b5ff54d664 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ tx.exe ##Coverage output coverage + +build diff --git a/build.js b/build.js index a5174d2c70..a0250dfb37 100644 --- a/build.js +++ b/build.js @@ -5,7 +5,7 @@ var winston = require('winston'); var buildStart; -var valid = ['js', 'clientCSS', 'acpCSS', 'tpl']; +var valid = ['js', 'clientCSS', 'acpCSS', 'tpl', 'lang']; exports.buildAll = function (callback) { exports.build(valid.join(','), callback); @@ -88,6 +88,12 @@ exports.buildTargets = function (targets, callback) { startTime = Date.now(); meta.templates.compile(step.bind(this, startTime, target, next)); break; + + case 'lang': + winston.info('[build] Building language files'); + startTime = Date.now(); + meta.languages.build(step.bind(this, startTime, target, next)); + break; default: winston.warn('[build] Unknown build target: \'' + target + '\''); diff --git a/public/src/modules/translator.js b/public/src/modules/translator.js index e4d74b5508..1e71d579e7 100644 --- a/public/src/modules/translator.js +++ b/public/src/modules/translator.js @@ -3,7 +3,7 @@ (function (factory) { 'use strict'; function loadClient(language, namespace) { - return Promise.resolve(jQuery.getJSON(config.relative_path + '/api/language/' + language + '/' + namespace)); + return Promise.resolve(jQuery.getJSON(config.relative_path + '/public/language/' + language + '/' + namespace + '.json')); } var warn = function () {}; if (typeof config === 'object' && config.environment === 'development') { @@ -17,7 +17,6 @@ } else if (typeof module === 'object' && module.exports) { // Node (function () { - require('promise-polyfill'); var languages = require('../../../src/languages'); if (global.env === 'development') { @@ -292,7 +291,7 @@ warn('[translator] Parameter `namespace` is ' + namespace + (namespace === '' ? '(empty string)' : '')); translation = Promise.resolve({}); } else { - translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace); + translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace).catch(function () { return {}; }); } if (key) { diff --git a/src/admin/search.js b/src/admin/search.js index 0bd140e3ba..699f0b33fb 100644 --- a/src/admin/search.js +++ b/src/admin/search.js @@ -5,7 +5,6 @@ var path = require('path'); var async = require('async'); var sanitizeHTML = require('sanitize-html'); -var languages = require('../languages'); var utils = require('../../public/src/utils'); var Translator = require('../../public/src/modules/translator').Translator; @@ -107,6 +106,8 @@ function fallback(namespace, callback) { } function initDict(language, callback) { + var translator = Translator.create(language); + getAdminNamespaces(function (err, namespaces) { if (err) { return callback(err); @@ -115,7 +116,9 @@ function initDict(language, callback) { async.map(namespaces, function (namespace, cb) { async.waterfall([ function (next) { - languages.get(language, namespace, next); + translator.getTranslation(namespace).then(function (translations) { + next(null, translations); + }, next); }, function (translations, next) { if (!translations || !Object.keys(translations).length) { @@ -139,7 +142,7 @@ function initDict(language, callback) { title[1] + '/' + title[2] + ']]') : ''); } - Translator.create(language).translate(title).then(function (title) { + translator.translate(title).then(function (title) { next(null, { namespace: namespace, translations: str + '\n' + title, diff --git a/src/controllers/index.js b/src/controllers/index.js index e6b0d3753c..57943d7078 100644 --- a/src/controllers/index.js +++ b/src/controllers/index.js @@ -351,7 +351,6 @@ Controllers.ping = function (req, res) { Controllers.handle404 = function (req, res) { var relativePath = nconf.get('relative_path'); - var isLanguage = new RegExp('^' + relativePath + '/api/language/.*/.*'); var isClientScript = new RegExp('^' + relativePath + '\\/src\\/.+\\.js'); if (plugins.hasListeners('action:meta.override404')) { @@ -364,8 +363,6 @@ Controllers.handle404 = function (req, res) { if (isClientScript.test(req.url)) { res.type('text/javascript').status(200).send(''); - } else if (isLanguage.test(req.url)) { - res.status(200).json({}); } else if (req.path.startsWith(relativePath + '/uploads') || (req.get('accept') && req.get('accept').indexOf('text/html') === -1) || req.path === '/favicon.ico') { meta.errors.log404(req.path || ''); res.sendStatus(404); diff --git a/src/languages.js b/src/languages.js index f3b9aa5743..5374b9d87f 100644 --- a/src/languages.js +++ b/src/languages.js @@ -3,53 +3,27 @@ var fs = require('fs'); var path = require('path'); var async = require('async'); -var LRU = require('lru-cache'); - -var plugins = require('./plugins'); var Languages = {}; -var languagesPath = path.join(__dirname, '../public/language'); +var languagesPath = path.join(__dirname, '../build/public/language'); Languages.init = function (next) { - if (Languages.hasOwnProperty('_cache')) { - Languages._cache.reset(); - } else { - Languages._cache = LRU(100); - } - next(); }; Languages.get = function (language, namespace, callback) { - var langNamespace = language + '/' + namespace; - - if (Languages._cache && Languages._cache.has(langNamespace)) { - return callback(null, Languages._cache.get(langNamespace)); - } - - var languageData; - fs.readFile(path.join(languagesPath, language, namespace + '.json'), { encoding: 'utf-8' }, function (err, data) { - if (err && err.code !== 'ENOENT') { + if (err) { return callback(err); } - // If language file in core cannot be read, then no language file present try { - languageData = JSON.parse(data) || {}; + data = JSON.parse(data) || {}; } catch (e) { - languageData = {}; + return callback(e); } - if (plugins.customLanguages.hasOwnProperty(langNamespace)) { - Object.assign(languageData, plugins.customLanguages[langNamespace]); - } - - if (Languages._cache) { - Languages._cache.set(langNamespace, languageData); - } - - callback(null, languageData); + callback(null, data); }); }; @@ -73,11 +47,13 @@ Languages.list = function (callback) { var configPath = path.join(languagesPath, folder, 'language.json'); - fs.readFile(configPath, function (err, stream) { - if (err) { + fs.readFile(configPath, function (err, buffer) { + if (err && err.code !== 'ENOENT') { return next(err); } - languages.push(JSON.parse(stream.toString())); + if (buffer) { + languages.push(JSON.parse(buffer.toString())); + } next(); }); }); diff --git a/src/meta.js b/src/meta.js index c732de15f4..2333e54d89 100644 --- a/src/meta.js +++ b/src/meta.js @@ -23,6 +23,7 @@ var utils = require('../public/src/utils'); require('./meta/dependencies')(Meta); Meta.templates = require('./meta/templates'); Meta.blacklist = require('./meta/blacklist'); + Meta.languages = require('./meta/languages'); /* Assorted */ Meta.userOrGroupExists = function (slug, callback) { diff --git a/src/meta/languages.js b/src/meta/languages.js new file mode 100644 index 0000000000..fca4473b0b --- /dev/null +++ b/src/meta/languages.js @@ -0,0 +1,142 @@ +'use strict'; + +var winston = require('winston'); +var path = require('path'); +var async = require('async'); +var fs = require('fs'); +var mkdirp = require('mkdirp'); + +var file = require('../file'); +var utils = require('../../public/src/utils'); +var Plugins = require('../plugins'); +var db = require('../database'); + +var buildLanguagesPath = path.join(__dirname, '../../build/public/language'); +var coreLanguagesPath = path.join(__dirname, '../../public/language'); + +function extrude(languageDir, paths) { + return paths.map(function (p) { + var rel = p.split(languageDir)[1].split(/[\/\\]/).slice(1); + return { + language: rel.shift().replace('_', '-').replace('@', '-x-'), + namespace: rel.join('/').replace(/\.json$/, ''), + path: p, + }; + }); +} + +function getTranslationTree(callback) { + async.waterfall([ + function (next) { + db.getSortedSetRange('plugins:active', 0, -1, next); + }, + function (plugins, next) { + var pluginBasePath = path.join(__dirname, '../../node_modules'); + var paths = plugins.map(function (plugin) { + return path.join(pluginBasePath, plugin); + }); + + // Filter out plugins with invalid paths + async.filter(paths, file.exists, function (paths) { + next(null, paths); + }); + }, + function (paths, next) { + async.map(paths, Plugins.loadPluginInfo, next); + }, + function (plugins, next) { + async.parallel({ + corePaths: function (cb) { + utils.walk(coreLanguagesPath, function (err, paths) { + if (err) { + return cb(err); + } + + cb(null, extrude(coreLanguagesPath, paths)); + }); + }, + pluginPaths: function (nxt) { + plugins = plugins.filter(function (pluginData) { + return (typeof pluginData.languages === 'string'); + }); + async.map(plugins, function (pluginData, cb) { + var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); + utils.walk(pathToFolder, function (err, paths) { + if (err) { + return cb(err); + } + + cb(null, extrude(pathToFolder, paths)); + }); + }, nxt); + } + }, next); + }, + function (data, next) { + var paths = data.pluginPaths.concat.apply([], data.pluginPaths); + paths = data.corePaths.concat(paths); + paths = paths.filter(function (p) { + return p.language && p.namespace && p.path; + }); + + var tree = {}; + + async.eachLimit(paths, 1000, function (data, cb) { + fs.readFile(data.path, function (err, file) { + if (err) { + return cb(err); + } + + try { + var obj = JSON.parse(file.toString()); + + tree[data.language] = tree[data.language] || {}; + tree[data.language][data.namespace] = tree[data.language][data.namespace] || {}; + Object.assign(tree[data.language][data.namespace], obj); + + cb(); + } catch (e) { + winston.warn('[build] Invalid JSON file at `' + data.path + '`'); + cb(); + } + }); + }, function (err) { + next(err, tree); + }); + } + ], callback); +} + +function writeLanguageFiles(tree, callback) { + async.eachLimit(Object.keys(tree), 10, function (language, cb) { + var namespaces = tree[language]; + async.eachLimit(Object.keys(namespaces), 100, function (namespace, next) { + var translations = namespaces[namespace]; + + var filePath = path.join(buildLanguagesPath, language, namespace + '.json'); + + mkdirp(path.dirname(filePath), function (err) { + if (err) { + return next(err); + } + + fs.writeFile(filePath, JSON.stringify(translations), next); + }); + }, cb); + }, callback); +} + +module.exports = { + build: function buildLanguages(callback) { + async.waterfall([ + getTranslationTree, + writeLanguageFiles, + ], function (err) { + if (err) { + winston.error('[build] Language build failed'); + throw err; + } + callback(); + }); + }, +}; diff --git a/src/middleware/index.js b/src/middleware/index.js index f48aaef748..c36f5767ed 100644 --- a/src/middleware/index.js +++ b/src/middleware/index.js @@ -183,23 +183,6 @@ middleware.applyBlacklist = function (req, res, next) { }); }; -middleware.getTranslation = function (req, res, next) { - var language = req.params.language; - var namespace = req.params[0]; - - if (language && namespace) { - languages.get(language, namespace, function (err, translations) { - if (err) { - return next(err); - } - - res.status(200).json(translations); - }); - } else { - res.status(404).json('{}'); - } -}; - middleware.processTimeagoLocales = function (req, res, next) { var fallback = req.path.indexOf('-short') === -1 ? 'jquery.timeago.en.js' : 'jquery.timeago.en-short.js', localPath = path.join(__dirname, '../../public/vendor/jquery/timeago/locales', req.path), diff --git a/src/middleware/maintenance.js b/src/middleware/maintenance.js index 5199fb9332..31ba25e250 100644 --- a/src/middleware/maintenance.js +++ b/src/middleware/maintenance.js @@ -24,7 +24,7 @@ module.exports = function (middleware) { '^/templates/[\\w/]+.tpl', '^/api/login', '^/api/widgets/render', - '^/api/language/.+', + '^/public/language', '^/uploads/system/site-logo.png' ]; diff --git a/src/plugins.js b/src/plugins.js index b8ddbdc0fe..789c92b72c 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -29,8 +29,6 @@ var middleware; Plugins.lessFiles = []; Plugins.clientScripts = []; Plugins.acpScripts = []; - Plugins.customLanguages = {}; - Plugins.customLanguageFallbacks = {}; Plugins.libraryPaths = []; Plugins.versionWarning = []; Plugins.languageCodes = []; diff --git a/src/plugins/load.js b/src/plugins/load.js index 60d584f99f..3121f8457e 100644 --- a/src/plugins/load.js +++ b/src/plugins/load.js @@ -9,8 +9,6 @@ var winston = require('winston'); var nconf = require('nconf'); var _ = require('underscore'); var file = require('../file'); - -var utils = require('../../public/src/utils'); var meta = require('../meta'); @@ -93,9 +91,6 @@ module.exports = function (Plugins) { function (next) { mapClientModules(pluginData, next); }, - function (next) { - loadLanguages(pluginData, next); - } ], function (err) { if (err) { winston.verbose('[plugins] Could not load plugin : ' + pluginData.id); @@ -254,60 +249,6 @@ module.exports = function (Plugins) { callback(); } - function loadLanguages(pluginData, callback) { - if (typeof pluginData.languages !== 'string') { - return callback(); - } - - var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); - var defaultLang = (pluginData.defaultLang || 'en_GB').replace('_', '-').replace('@', '-x-'); - - utils.walk(pathToFolder, function (err, languages) { - if (err) { - return callback(err); - } - - async.each(languages, function (pathToLang, next) { - fs.readFile(pathToLang, function (err, file) { - if (err) { - return next(err); - } - var data; - var language = path.dirname(pathToLang).split(/[\/\\]/).pop().replace('_', '-').replace('@', '-x-'); - var namespace = path.basename(pathToLang, '.json'); - var langNamespace = language + '/' + namespace; - - try { - data = JSON.parse(file.toString()); - } catch (err) { - winston.error('[plugins] Unable to parse custom language file: ' + pathToLang + '\r\n' + err.stack); - return next(err); - } - - Plugins.customLanguages[langNamespace] = Plugins.customLanguages[langNamespace] || {}; - Object.assign(Plugins.customLanguages[langNamespace], data); - - if (defaultLang && defaultLang === language) { - Plugins.languageCodes.filter(function (lang) { - return defaultLang !== lang; - }).forEach(function (lang) { - var langNS = lang + '/' + namespace; - Plugins.customLanguages[langNS] = Object.assign(Plugins.customLanguages[langNS] || {}, data); - }); - } - - next(); - }); - }, function (err) { - if (err) { - return callback(err); - } - - callback(); - }); - }); - } - function resolveModulePath(fullPath, relPath) { /** * With npm@3, dependencies can become flattened, and appear at the root level. @@ -365,6 +306,7 @@ module.exports = function (Plugins) { return callback(new Error('[[error:parse-error]]')); } + callback(null, pluginData); }); }; diff --git a/src/routes/index.js b/src/routes/index.js index 863715008e..ea86434570 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -144,7 +144,17 @@ module.exports = function (app, middleware, hotswapIds) { } app.use(middleware.privateUploads); - app.use(relativePath + '/api/language/:language/(([a-zA-Z0-9\\-_.\\/]+))', middleware.getTranslation); + app.use(relativePath + '/public/language', express.static(path.join(__dirname, '../../', 'build/public/language'), { + maxAge: app.enabled('cache') ? 5184000000 : 0 + })); + + // DEPRECATED + app.use(relativePath + '/api/language', function (req, res) { + winston.warn('[deprecated] Accessing language files from `/api/language` is deprecated. ' + + 'Use `/public/language/[langCode]/[namespace].json` for prefetch paths.'); + res.redirect(relativePath + '/public/language' + req.path + '.json'); + }); + app.use(relativePath, express.static(path.join(__dirname, '../../', 'public'), { maxAge: app.enabled('cache') ? 5184000000 : 0 })); From c97764ee6fbb87392085fa386acda5a6b24441d3 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 12 Jan 2017 20:25:19 -0700 Subject: [PATCH 03/10] Fix incomplete "group" search results --- src/admin/search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/search.js b/src/admin/search.js index 699f0b33fb..860c527e6b 100644 --- a/src/admin/search.js +++ b/src/admin/search.js @@ -18,7 +18,7 @@ function filterDirectories(directories) { // exclude category.tpl, group.tpl, category-analytics.tpl return !dir.includes('/partials/') && /\/.*\//.test(dir) && - !/category|group|category\-analytics$/.test(dir); + !/manage\/(category|group|category\-analytics)$/.test(dir); }); } From b4b3be3d976d4918eab0e92c51e221b390f7ff8b Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Fri, 13 Jan 2017 21:12:20 -0700 Subject: [PATCH 04/10] Cache bust translations, query param name --- public/src/modules/translator.js | 2 +- public/src/require-config.js | 2 +- src/meta/configs.js | 2 +- src/routes/index.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/public/src/modules/translator.js b/public/src/modules/translator.js index 1e71d579e7..a146a94eeb 100644 --- a/public/src/modules/translator.js +++ b/public/src/modules/translator.js @@ -3,7 +3,7 @@ (function (factory) { 'use strict'; function loadClient(language, namespace) { - return Promise.resolve(jQuery.getJSON(config.relative_path + '/public/language/' + language + '/' + namespace + '.json')); + return Promise.resolve(jQuery.getJSON(config.relative_path + '/public/language/' + language + '/' + namespace + '.json?' + config['cache-buster'])); } var warn = function () {}; if (typeof config === 'object' && config.environment === 'development') { diff --git a/public/src/require-config.js b/public/src/require-config.js index 0ad2f8a58c..8618685052 100644 --- a/public/src/require-config.js +++ b/public/src/require-config.js @@ -1,7 +1,7 @@ require.config({ baseUrl: config.relative_path + "/src/modules", waitSeconds: 7, - urlArgs: "v=" + config['cache-buster'], + urlArgs: config['cache-buster'], paths: { 'forum': '../client', 'admin': '../admin', diff --git a/src/meta/configs.js b/src/meta/configs.js index 0ad2bf2545..75da0595d2 100644 --- a/src/meta/configs.js +++ b/src/meta/configs.js @@ -21,7 +21,7 @@ module.exports = function (Meta) { Meta.configs.list(next); }, function (config, next) { - config['cache-buster'] = utils.generateUUID(); + config['cache-buster'] = 'v=' + utils.generateUUID(); Meta.config = config; setImmediate(next); diff --git a/src/routes/index.js b/src/routes/index.js index ea86434570..c4d61ed4ec 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -144,7 +144,7 @@ module.exports = function (app, middleware, hotswapIds) { } app.use(middleware.privateUploads); - app.use(relativePath + '/public/language', express.static(path.join(__dirname, '../../', 'build/public/language'), { + app.use(relativePath + '/public', express.static(path.join(__dirname, '../../', 'build/public'), { maxAge: app.enabled('cache') ? 5184000000 : 0 })); From e1a29f4aa8c56c895592dac45b31051ea6e6853d Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Sat, 14 Jan 2017 16:27:35 -0700 Subject: [PATCH 05/10] Switch to `assets` route --- public/src/modules/translator.js | 2 +- src/routes/index.js | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/public/src/modules/translator.js b/public/src/modules/translator.js index a146a94eeb..eac467f413 100644 --- a/public/src/modules/translator.js +++ b/public/src/modules/translator.js @@ -3,7 +3,7 @@ (function (factory) { 'use strict'; function loadClient(language, namespace) { - return Promise.resolve(jQuery.getJSON(config.relative_path + '/public/language/' + language + '/' + namespace + '.json?' + config['cache-buster'])); + return Promise.resolve(jQuery.getJSON(config.relative_path + '/assets/language/' + language + '/' + namespace + '.json?' + config['cache-buster'])); } var warn = function () {}; if (typeof config === 'object' && config.environment === 'development') { diff --git a/src/routes/index.js b/src/routes/index.js index c4d61ed4ec..158774b624 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -4,6 +4,7 @@ var nconf = require('nconf'); var winston = require('winston'); var path = require('path'); var async = require('async'); +var meta = require('../meta'); var controllers = require('../controllers'); var plugins = require('../plugins'); var user = require('../user'); @@ -144,15 +145,15 @@ module.exports = function (app, middleware, hotswapIds) { } app.use(middleware.privateUploads); - app.use(relativePath + '/public', express.static(path.join(__dirname, '../../', 'build/public'), { + app.use(relativePath + '/assets', express.static(path.join(__dirname, '../../', 'build/public'), { maxAge: app.enabled('cache') ? 5184000000 : 0 })); // DEPRECATED app.use(relativePath + '/api/language', function (req, res) { winston.warn('[deprecated] Accessing language files from `/api/language` is deprecated. ' + - 'Use `/public/language/[langCode]/[namespace].json` for prefetch paths.'); - res.redirect(relativePath + '/public/language' + req.path + '.json'); + 'Use `/assets/language/[langCode]/[namespace].json` for prefetch paths.'); + res.redirect(relativePath + '/assets/language' + req.path + '.json?' + meta.config['cache-buster']); }); app.use(relativePath, express.static(path.join(__dirname, '../../', 'public'), { From aff53cccc5e64ee9668dda8fbc474cd8f87d0fdc Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Mon, 16 Jan 2017 16:31:16 -0700 Subject: [PATCH 06/10] Plugin language fallback support --- src/meta/languages.js | 158 +++++++++++++++++++++++++++++------------- 1 file changed, 110 insertions(+), 48 deletions(-) diff --git a/src/meta/languages.js b/src/meta/languages.js index fca4473b0b..91e14767e2 100644 --- a/src/meta/languages.js +++ b/src/meta/languages.js @@ -14,17 +14,6 @@ var db = require('../database'); var buildLanguagesPath = path.join(__dirname, '../../build/public/language'); var coreLanguagesPath = path.join(__dirname, '../../public/language'); -function extrude(languageDir, paths) { - return paths.map(function (p) { - var rel = p.split(languageDir)[1].split(/[\/\\]/).slice(1); - return { - language: rel.shift().replace('_', '-').replace('@', '-x-'), - namespace: rel.join('/').replace(/\.json$/, ''), - path: p, - }; - }); -} - function getTranslationTree(callback) { async.waterfall([ function (next) { @@ -45,65 +34,138 @@ function getTranslationTree(callback) { async.map(paths, Plugins.loadPluginInfo, next); }, function (plugins, next) { - async.parallel({ - corePaths: function (cb) { + var languages = [], namespaces = []; + + function extrude(languageDir, paths) { + paths.forEach(function (p) { + var rel = p.split(languageDir)[1].split(/[\/\\]/).slice(1); + var language = rel.shift().replace('_', '-').replace('@', '-x-'); + var namespace = rel.join('/').replace(/\.json$/, ''); + + if (!language || !namespace) { + return; + } + + if (!languages.includes(language)) { + languages.push(language); + } + if (!namespaces.includes(namespace)) { + namespaces.push(namespace); + } + }); + } + + plugins = plugins.filter(function (pluginData) { + return (typeof pluginData.languages === 'string'); + }); + async.parallel([ + function (nxt) { utils.walk(coreLanguagesPath, function (err, paths) { if (err) { - return cb(err); + return nxt(err); } - cb(null, extrude(coreLanguagesPath, paths)); + extrude(coreLanguagesPath, paths); + nxt(); }); }, - pluginPaths: function (nxt) { - plugins = plugins.filter(function (pluginData) { - return (typeof pluginData.languages === 'string'); - }); - async.map(plugins, function (pluginData, cb) { + function (nxt) { + async.each(plugins, function (pluginData, cb) { var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); utils.walk(pathToFolder, function (err, paths) { if (err) { return cb(err); } - cb(null, extrude(pathToFolder, paths)); + extrude(pathToFolder, paths); + cb(); }); }, nxt); + }, + ], function (err) { + if (err) { + return next(err); } - }, next); - }, - function (data, next) { - var paths = data.pluginPaths.concat.apply([], data.pluginPaths); - paths = data.corePaths.concat(paths); - paths = paths.filter(function (p) { - return p.language && p.namespace && p.path; + + next(null, { + languages: languages, + namespaces: namespaces, + plugins: plugins, + }); }); + }, + function (ref, next) { + var languages = ref.languages; + var namespaces = ref.namespaces; + var plugins = ref.plugins; var tree = {}; - - async.eachLimit(paths, 1000, function (data, cb) { - fs.readFile(data.path, function (err, file) { - if (err) { - return cb(err); - } - try { - var obj = JSON.parse(file.toString()); - - tree[data.language] = tree[data.language] || {}; - tree[data.language][data.namespace] = tree[data.language][data.namespace] || {}; - Object.assign(tree[data.language][data.namespace], obj); - - cb(); - } catch (e) { - winston.warn('[build] Invalid JSON file at `' + data.path + '`'); - cb(); - } - }); + async.eachLimit(languages, 10, function (lang, nxt) { + async.eachLimit(namespaces, 10, function (ns, cb) { + var translations = {}; + async.series([ + function (n) { + fs.readFile(path.join(coreLanguagesPath, lang, ns + '.json'), function (err, buffer) { + if (err) { + if (err.code === 'ENOENT') { + return n(); + } + return n(err); + } + + try { + Object.assign(translations, JSON.parse(buffer.toString())); + n(); + } catch (err) { + n(err); + } + }); + }, + function (n) { + async.eachLimit(plugins, 10, function (pluginData, call) { + var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); + function tryLang(lang, onEnoent) { + fs.readFile(path.join(pluginLanguages, lang, ns + '.json'), function (err, buffer) { + if (err) { + if (err.code === 'ENOENT') { + return onEnoent(); + } + return call(err); + } + + try { + Object.assign(translations, JSON.parse(buffer.toString())); + call(); + } catch (err) { + call(err); + } + }); + } + + tryLang(lang, function () { + tryLang(lang.replace('-', '_').replace('-x-', '@'), function () { + tryLang(pluginData.defaultLang, function () { + tryLang(pluginData.defaultLang.replace('-', '_').replace('-x-', '@'), call); + }); + }); + }); + }, function (err) { + if (err) { + return n(err); + } + + tree[lang] = tree[lang] || {}; + tree[lang][ns] = translations; + n(); + }); + }, + ], cb); + }, nxt); }, function (err) { next(err, tree); }); - } + }, ], callback); } From 65b866192ea3d3a81c56729058e8b141cfd794d7 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Mon, 16 Jan 2017 16:48:56 -0700 Subject: [PATCH 07/10] Fix failing tests --- src/meta/languages.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/meta/languages.js b/src/meta/languages.js index 91e14767e2..9b3a576bc6 100644 --- a/src/meta/languages.js +++ b/src/meta/languages.js @@ -46,10 +46,10 @@ function getTranslationTree(callback) { return; } - if (!languages.includes(language)) { + if (languages.indexOf(language) === -1) { languages.push(language); } - if (!namespaces.includes(namespace)) { + if (namespaces.indexOf(namespace) === -1) { namespaces.push(namespace); } }); From e94d47976dc5d8018877c977c3f4df085172532b Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Tue, 17 Jan 2017 20:22:38 -0700 Subject: [PATCH 08/10] Build language files on change --- Gruntfile.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Gruntfile.js b/Gruntfile.js index be761a16cf..99a103a86d 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -27,6 +27,8 @@ module.exports = function (grunt) { compiling = 'js'; } else if (target === 'templatesUpdated') { compiling = 'tpl'; + } else if (target === 'langUpdated') { + compiling = 'lang'; } else if (target === 'serverUpdated') { // Do nothing, just restart } @@ -93,7 +95,15 @@ module.exports = function (grunt) { '!node_modules/nodebb-*/node_modules/**', '!node_modules/nodebb-*/.git/**' ] - } + }, + langUpdated: { + files: [ + 'public/language/**/*.json', + 'node_modules/nodebb-*/**/*.json', + '!node_modules/nodebb-*/node_modules/**', + '!node_modules/nodebb-*/.git/**', + ], + }, } }); From 7f2ded17abbbacc66295323307c29c93fb51185e Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Tue, 17 Jan 2017 20:41:04 -0700 Subject: [PATCH 09/10] Comments, misc --- src/meta/languages.js | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/src/meta/languages.js b/src/meta/languages.js index 9b3a576bc6..563b34889c 100644 --- a/src/meta/languages.js +++ b/src/meta/languages.js @@ -16,6 +16,7 @@ var coreLanguagesPath = path.join(__dirname, '../../public/language'); function getTranslationTree(callback) { async.waterfall([ + // get plugin data function (next) { db.getSortedSetRange('plugins:active', 0, -1, next); }, @@ -33,9 +34,12 @@ function getTranslationTree(callback) { function (paths, next) { async.map(paths, Plugins.loadPluginInfo, next); }, + + // generate list of languages and namespaces function (plugins, next) { var languages = [], namespaces = []; + // pull languages and namespaces from paths function extrude(languageDir, paths) { paths.forEach(function (p) { var rel = p.split(languageDir)[1].split(/[\/\\]/).slice(1); @@ -59,6 +63,7 @@ function getTranslationTree(callback) { return (typeof pluginData.languages === 'string'); }); async.parallel([ + // get core languages and namespaces function (nxt) { utils.walk(coreLanguagesPath, function (err, paths) { if (err) { @@ -69,6 +74,7 @@ function getTranslationTree(callback) { nxt(); }); }, + // get plugin languages and namespaces function (nxt) { async.each(plugins, function (pluginData, cb) { var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); @@ -94,6 +100,10 @@ function getTranslationTree(callback) { }); }); }, + + // for each language and namespace combination, + // run through core and all plugins to generate + // a full translation hash function (ref, next) { var languages = ref.languages; var namespaces = ref.namespaces; @@ -104,7 +114,9 @@ function getTranslationTree(callback) { async.eachLimit(languages, 10, function (lang, nxt) { async.eachLimit(namespaces, 10, function (ns, cb) { var translations = {}; + async.series([ + // core first function (n) { fs.readFile(path.join(coreLanguagesPath, lang, ns + '.json'), function (err, buffer) { if (err) { @@ -123,6 +135,11 @@ function getTranslationTree(callback) { }); }, function (n) { + // for each plugin, fallback in this order: + // 1. correct language string (en-GB) + // 2. old language string (en_GB) + // 3. plugin defaultLang (en-US) + // 4. old plugin defaultLang (en_US) async.eachLimit(plugins, 10, function (pluginData, call) { var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); function tryLang(lang, onEnoent) { @@ -169,7 +186,9 @@ function getTranslationTree(callback) { ], callback); } +// write translation hashes from the generated tree to language files function writeLanguageFiles(tree, callback) { + // iterate over languages and namespaces async.eachLimit(Object.keys(tree), 10, function (language, cb) { var namespaces = tree[language]; async.eachLimit(Object.keys(namespaces), 100, function (namespace, next) { @@ -188,17 +207,15 @@ function writeLanguageFiles(tree, callback) { }, callback); } -module.exports = { - build: function buildLanguages(callback) { - async.waterfall([ - getTranslationTree, - writeLanguageFiles, - ], function (err) { - if (err) { - winston.error('[build] Language build failed'); - throw err; - } - callback(); - }); - }, +exports.build = function buildLanguages(callback) { + async.waterfall([ + getTranslationTree, + writeLanguageFiles, + ], function (err) { + if (err) { + winston.error('[build] Language build failed: ' + err.message); + throw err; + } + callback(); + }); }; From 80b4c6fa684fe92d1510b0687bc56f9e690c0495 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 19 Jan 2017 10:53:25 -0700 Subject: [PATCH 10/10] Exclude config files --- Gruntfile.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Gruntfile.js b/Gruntfile.js index 99a103a86d..66f12bba3e 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -102,6 +102,9 @@ module.exports = function (grunt) { 'node_modules/nodebb-*/**/*.json', '!node_modules/nodebb-*/node_modules/**', '!node_modules/nodebb-*/.git/**', + '!node_modules/nodebb-*/plugin.json', + '!node_modules/nodebb-*/package.json', + '!node_modules/nodebb-*/theme.json', ], }, }