diff --git a/.gitignore b/.gitignore index 5f402cd667..b5ff54d664 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ tx.exe ##Coverage output coverage + +build diff --git a/Gruntfile.js b/Gruntfile.js index be761a16cf..66f12bba3e 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -27,6 +27,8 @@ module.exports = function (grunt) { compiling = 'js'; } else if (target === 'templatesUpdated') { compiling = 'tpl'; + } else if (target === 'langUpdated') { + compiling = 'lang'; } else if (target === 'serverUpdated') { // Do nothing, just restart } @@ -93,7 +95,18 @@ module.exports = function (grunt) { '!node_modules/nodebb-*/node_modules/**', '!node_modules/nodebb-*/.git/**' ] - } + }, + langUpdated: { + files: [ + 'public/language/**/*.json', + 'node_modules/nodebb-*/**/*.json', + '!node_modules/nodebb-*/node_modules/**', + '!node_modules/nodebb-*/.git/**', + '!node_modules/nodebb-*/plugin.json', + '!node_modules/nodebb-*/package.json', + '!node_modules/nodebb-*/theme.json', + ], + }, } }); diff --git a/build.js b/build.js index a5174d2c70..a0250dfb37 100644 --- a/build.js +++ b/build.js @@ -5,7 +5,7 @@ var winston = require('winston'); var buildStart; -var valid = ['js', 'clientCSS', 'acpCSS', 'tpl']; +var valid = ['js', 'clientCSS', 'acpCSS', 'tpl', 'lang']; exports.buildAll = function (callback) { exports.build(valid.join(','), callback); @@ -88,6 +88,12 @@ exports.buildTargets = function (targets, callback) { startTime = Date.now(); meta.templates.compile(step.bind(this, startTime, target, next)); break; + + case 'lang': + winston.info('[build] Building language files'); + startTime = Date.now(); + meta.languages.build(step.bind(this, startTime, target, next)); + break; default: winston.warn('[build] Unknown build target: \'' + target + '\''); diff --git a/nodebb b/nodebb index 62115e110b..e1a4ab7fbf 100755 --- a/nodebb +++ b/nodebb @@ -375,7 +375,7 @@ switch(process.argv[2]) { async.series([ function (next) { process.stdout.write('1. '.bold + 'Bringing base dependencies up to date... '.yellow); - require('child_process').execFile('/usr/bin/env', ['npm', 'i', '--production'], { stdio: 'ignore' }, next); + cproc.exec('npm i --production', { cwd: __dirname, stdio: 'ignore' }, next); }, function (next) { process.stdout.write('OK\n'.green); diff --git a/public/src/client/category.js b/public/src/client/category.js index 4afabb5093..24b0d274f6 100644 --- a/public/src/client/category.js +++ b/public/src/client/category.js @@ -103,12 +103,9 @@ define('forum/category', [ return bottomIndex; }; - $(window).on('action:popstate', function (ev, data) { - if (data.url.startsWith('category/')) { - var cid = data.url.match(/^category\/(\d+)/); - if (cid && cid[1]) { - cid = cid[1]; - } + $(window).on('action:ajaxify.contentLoaded', function (ev, data) { + if (ajaxify.data.template.category) { + var cid = ajaxify.data.cid; if (!cid) { return; } @@ -140,7 +137,9 @@ define('forum/category', [ $('[component="category"]').empty(); loadTopicsAfter(Math.max(0, bookmarkIndex - 1) + 1, 1, function () { - Category.scrollToTopic(bookmarkIndex, clickedIndex, 0); + $(window).one('action:topics.loaded', function () { + Category.scrollToTopic(bookmarkIndex, clickedIndex, 0); + }); }); } } @@ -167,9 +166,8 @@ define('forum/category', [ } var scrollTo = components.get('category/topic', 'index', bookmarkIndex); - var cid = ajaxify.data.cid; - if (scrollTo.length && cid) { + if (scrollTo.length) { $('html, body').animate({ scrollTop: (scrollTo.offset().top - offset) + 'px' }, duration !== undefined ? duration : 400, function () { @@ -272,7 +270,7 @@ define('forum/category', [ return callback(); } - $(window).trigger('action:categories.loading'); + $(window).trigger('action:category.loading'); var params = utils.params(); infinitescroll.loadMore('categories.loadMore', { cid: ajaxify.data.cid, @@ -288,7 +286,7 @@ define('forum/category', [ done(); } - $(window).trigger('action:categories.loaded'); + $(window).trigger('action:category.loaded'); callback(); }); } diff --git a/public/src/modules/translator.js b/public/src/modules/translator.js index e4d74b5508..eac467f413 100644 --- a/public/src/modules/translator.js +++ b/public/src/modules/translator.js @@ -3,7 +3,7 @@ (function (factory) { 'use strict'; function loadClient(language, namespace) { - return Promise.resolve(jQuery.getJSON(config.relative_path + '/api/language/' + language + '/' + namespace)); + return Promise.resolve(jQuery.getJSON(config.relative_path + '/assets/language/' + language + '/' + namespace + '.json?' + config['cache-buster'])); } var warn = function () {}; if (typeof config === 'object' && config.environment === 'development') { @@ -17,7 +17,6 @@ } else if (typeof module === 'object' && module.exports) { // Node (function () { - require('promise-polyfill'); var languages = require('../../../src/languages'); if (global.env === 'development') { @@ -292,7 +291,7 @@ warn('[translator] Parameter `namespace` is ' + namespace + (namespace === '' ? '(empty string)' : '')); translation = Promise.resolve({}); } else { - translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace); + translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace).catch(function () { return {}; }); } if (key) { diff --git a/public/src/require-config.js b/public/src/require-config.js index 0ad2f8a58c..8618685052 100644 --- a/public/src/require-config.js +++ b/public/src/require-config.js @@ -1,7 +1,7 @@ require.config({ baseUrl: config.relative_path + "/src/modules", waitSeconds: 7, - urlArgs: "v=" + config['cache-buster'], + urlArgs: config['cache-buster'], paths: { 'forum': '../client', 'admin': '../admin', diff --git a/src/admin/search.js b/src/admin/search.js index 0bd140e3ba..860c527e6b 100644 --- a/src/admin/search.js +++ b/src/admin/search.js @@ -5,7 +5,6 @@ var path = require('path'); var async = require('async'); var sanitizeHTML = require('sanitize-html'); -var languages = require('../languages'); var utils = require('../../public/src/utils'); var Translator = require('../../public/src/modules/translator').Translator; @@ -19,7 +18,7 @@ function filterDirectories(directories) { // exclude category.tpl, group.tpl, category-analytics.tpl return !dir.includes('/partials/') && /\/.*\//.test(dir) && - !/category|group|category\-analytics$/.test(dir); + !/manage\/(category|group|category\-analytics)$/.test(dir); }); } @@ -107,6 +106,8 @@ function fallback(namespace, callback) { } function initDict(language, callback) { + var translator = Translator.create(language); + getAdminNamespaces(function (err, namespaces) { if (err) { return callback(err); @@ -115,7 +116,9 @@ function initDict(language, callback) { async.map(namespaces, function (namespace, cb) { async.waterfall([ function (next) { - languages.get(language, namespace, next); + translator.getTranslation(namespace).then(function (translations) { + next(null, translations); + }, next); }, function (translations, next) { if (!translations || !Object.keys(translations).length) { @@ -139,7 +142,7 @@ function initDict(language, callback) { title[1] + '/' + title[2] + ']]') : ''); } - Translator.create(language).translate(title).then(function (title) { + translator.translate(title).then(function (title) { next(null, { namespace: namespace, translations: str + '\n' + title, diff --git a/src/controllers/index.js b/src/controllers/index.js index 3b22c7ad1f..7b97c878d6 100644 --- a/src/controllers/index.js +++ b/src/controllers/index.js @@ -353,7 +353,6 @@ Controllers.ping = function (req, res) { Controllers.handle404 = function (req, res) { var relativePath = nconf.get('relative_path'); - var isLanguage = new RegExp('^' + relativePath + '/api/language/.*/.*'); var isClientScript = new RegExp('^' + relativePath + '\\/src\\/.+\\.js'); if (plugins.hasListeners('action:meta.override404')) { @@ -366,8 +365,6 @@ Controllers.handle404 = function (req, res) { if (isClientScript.test(req.url)) { res.type('text/javascript').status(200).send(''); - } else if (isLanguage.test(req.url)) { - res.status(200).json({}); } else if (req.path.startsWith(relativePath + '/uploads') || (req.get('accept') && req.get('accept').indexOf('text/html') === -1) || req.path === '/favicon.ico') { meta.errors.log404(req.path || ''); res.sendStatus(404); diff --git a/src/languages.js b/src/languages.js index f3b9aa5743..5374b9d87f 100644 --- a/src/languages.js +++ b/src/languages.js @@ -3,53 +3,27 @@ var fs = require('fs'); var path = require('path'); var async = require('async'); -var LRU = require('lru-cache'); - -var plugins = require('./plugins'); var Languages = {}; -var languagesPath = path.join(__dirname, '../public/language'); +var languagesPath = path.join(__dirname, '../build/public/language'); Languages.init = function (next) { - if (Languages.hasOwnProperty('_cache')) { - Languages._cache.reset(); - } else { - Languages._cache = LRU(100); - } - next(); }; Languages.get = function (language, namespace, callback) { - var langNamespace = language + '/' + namespace; - - if (Languages._cache && Languages._cache.has(langNamespace)) { - return callback(null, Languages._cache.get(langNamespace)); - } - - var languageData; - fs.readFile(path.join(languagesPath, language, namespace + '.json'), { encoding: 'utf-8' }, function (err, data) { - if (err && err.code !== 'ENOENT') { + if (err) { return callback(err); } - // If language file in core cannot be read, then no language file present try { - languageData = JSON.parse(data) || {}; + data = JSON.parse(data) || {}; } catch (e) { - languageData = {}; + return callback(e); } - if (plugins.customLanguages.hasOwnProperty(langNamespace)) { - Object.assign(languageData, plugins.customLanguages[langNamespace]); - } - - if (Languages._cache) { - Languages._cache.set(langNamespace, languageData); - } - - callback(null, languageData); + callback(null, data); }); }; @@ -73,11 +47,13 @@ Languages.list = function (callback) { var configPath = path.join(languagesPath, folder, 'language.json'); - fs.readFile(configPath, function (err, stream) { - if (err) { + fs.readFile(configPath, function (err, buffer) { + if (err && err.code !== 'ENOENT') { return next(err); } - languages.push(JSON.parse(stream.toString())); + if (buffer) { + languages.push(JSON.parse(buffer.toString())); + } next(); }); }); diff --git a/src/meta.js b/src/meta.js index c732de15f4..2333e54d89 100644 --- a/src/meta.js +++ b/src/meta.js @@ -23,6 +23,7 @@ var utils = require('../public/src/utils'); require('./meta/dependencies')(Meta); Meta.templates = require('./meta/templates'); Meta.blacklist = require('./meta/blacklist'); + Meta.languages = require('./meta/languages'); /* Assorted */ Meta.userOrGroupExists = function (slug, callback) { diff --git a/src/meta/configs.js b/src/meta/configs.js index 0ad2bf2545..75da0595d2 100644 --- a/src/meta/configs.js +++ b/src/meta/configs.js @@ -21,7 +21,7 @@ module.exports = function (Meta) { Meta.configs.list(next); }, function (config, next) { - config['cache-buster'] = utils.generateUUID(); + config['cache-buster'] = 'v=' + utils.generateUUID(); Meta.config = config; setImmediate(next); diff --git a/src/meta/languages.js b/src/meta/languages.js new file mode 100644 index 0000000000..563b34889c --- /dev/null +++ b/src/meta/languages.js @@ -0,0 +1,221 @@ +'use strict'; + +var winston = require('winston'); +var path = require('path'); +var async = require('async'); +var fs = require('fs'); +var mkdirp = require('mkdirp'); + +var file = require('../file'); +var utils = require('../../public/src/utils'); +var Plugins = require('../plugins'); +var db = require('../database'); + +var buildLanguagesPath = path.join(__dirname, '../../build/public/language'); +var coreLanguagesPath = path.join(__dirname, '../../public/language'); + +function getTranslationTree(callback) { + async.waterfall([ + // get plugin data + function (next) { + db.getSortedSetRange('plugins:active', 0, -1, next); + }, + function (plugins, next) { + var pluginBasePath = path.join(__dirname, '../../node_modules'); + var paths = plugins.map(function (plugin) { + return path.join(pluginBasePath, plugin); + }); + + // Filter out plugins with invalid paths + async.filter(paths, file.exists, function (paths) { + next(null, paths); + }); + }, + function (paths, next) { + async.map(paths, Plugins.loadPluginInfo, next); + }, + + // generate list of languages and namespaces + function (plugins, next) { + var languages = [], namespaces = []; + + // pull languages and namespaces from paths + function extrude(languageDir, paths) { + paths.forEach(function (p) { + var rel = p.split(languageDir)[1].split(/[\/\\]/).slice(1); + var language = rel.shift().replace('_', '-').replace('@', '-x-'); + var namespace = rel.join('/').replace(/\.json$/, ''); + + if (!language || !namespace) { + return; + } + + if (languages.indexOf(language) === -1) { + languages.push(language); + } + if (namespaces.indexOf(namespace) === -1) { + namespaces.push(namespace); + } + }); + } + + plugins = plugins.filter(function (pluginData) { + return (typeof pluginData.languages === 'string'); + }); + async.parallel([ + // get core languages and namespaces + function (nxt) { + utils.walk(coreLanguagesPath, function (err, paths) { + if (err) { + return nxt(err); + } + + extrude(coreLanguagesPath, paths); + nxt(); + }); + }, + // get plugin languages and namespaces + function (nxt) { + async.each(plugins, function (pluginData, cb) { + var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); + utils.walk(pathToFolder, function (err, paths) { + if (err) { + return cb(err); + } + + extrude(pathToFolder, paths); + cb(); + }); + }, nxt); + }, + ], function (err) { + if (err) { + return next(err); + } + + next(null, { + languages: languages, + namespaces: namespaces, + plugins: plugins, + }); + }); + }, + + // for each language and namespace combination, + // run through core and all plugins to generate + // a full translation hash + function (ref, next) { + var languages = ref.languages; + var namespaces = ref.namespaces; + var plugins = ref.plugins; + + var tree = {}; + + async.eachLimit(languages, 10, function (lang, nxt) { + async.eachLimit(namespaces, 10, function (ns, cb) { + var translations = {}; + + async.series([ + // core first + function (n) { + fs.readFile(path.join(coreLanguagesPath, lang, ns + '.json'), function (err, buffer) { + if (err) { + if (err.code === 'ENOENT') { + return n(); + } + return n(err); + } + + try { + Object.assign(translations, JSON.parse(buffer.toString())); + n(); + } catch (err) { + n(err); + } + }); + }, + function (n) { + // for each plugin, fallback in this order: + // 1. correct language string (en-GB) + // 2. old language string (en_GB) + // 3. plugin defaultLang (en-US) + // 4. old plugin defaultLang (en_US) + async.eachLimit(plugins, 10, function (pluginData, call) { + var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); + function tryLang(lang, onEnoent) { + fs.readFile(path.join(pluginLanguages, lang, ns + '.json'), function (err, buffer) { + if (err) { + if (err.code === 'ENOENT') { + return onEnoent(); + } + return call(err); + } + + try { + Object.assign(translations, JSON.parse(buffer.toString())); + call(); + } catch (err) { + call(err); + } + }); + } + + tryLang(lang, function () { + tryLang(lang.replace('-', '_').replace('-x-', '@'), function () { + tryLang(pluginData.defaultLang, function () { + tryLang(pluginData.defaultLang.replace('-', '_').replace('-x-', '@'), call); + }); + }); + }); + }, function (err) { + if (err) { + return n(err); + } + + tree[lang] = tree[lang] || {}; + tree[lang][ns] = translations; + n(); + }); + }, + ], cb); + }, nxt); + }, function (err) { + next(err, tree); + }); + }, + ], callback); +} + +// write translation hashes from the generated tree to language files +function writeLanguageFiles(tree, callback) { + // iterate over languages and namespaces + async.eachLimit(Object.keys(tree), 10, function (language, cb) { + var namespaces = tree[language]; + async.eachLimit(Object.keys(namespaces), 100, function (namespace, next) { + var translations = namespaces[namespace]; + + var filePath = path.join(buildLanguagesPath, language, namespace + '.json'); + + mkdirp(path.dirname(filePath), function (err) { + if (err) { + return next(err); + } + + fs.writeFile(filePath, JSON.stringify(translations), next); + }); + }, cb); + }, callback); +} + +exports.build = function buildLanguages(callback) { + async.waterfall([ + getTranslationTree, + writeLanguageFiles, + ], function (err) { + if (err) { + winston.error('[build] Language build failed: ' + err.message); + throw err; + } + callback(); + }); +}; diff --git a/src/middleware/index.js b/src/middleware/index.js index 85e29a879a..8f8f8a8002 100644 --- a/src/middleware/index.js +++ b/src/middleware/index.js @@ -207,23 +207,6 @@ middleware.applyBlacklist = function (req, res, next) { }); }; -middleware.getTranslation = function (req, res, next) { - var language = req.params.language; - var namespace = req.params[0]; - - if (language && namespace) { - languages.get(language, namespace, function (err, translations) { - if (err) { - return next(err); - } - - res.status(200).json(translations); - }); - } else { - res.status(404).json('{}'); - } -}; - middleware.processTimeagoLocales = function (req, res, next) { var fallback = req.path.indexOf('-short') === -1 ? 'jquery.timeago.en.js' : 'jquery.timeago.en-short.js', localPath = path.join(__dirname, '../../public/vendor/jquery/timeago/locales', req.path), diff --git a/src/middleware/maintenance.js b/src/middleware/maintenance.js index 5199fb9332..31ba25e250 100644 --- a/src/middleware/maintenance.js +++ b/src/middleware/maintenance.js @@ -24,7 +24,7 @@ module.exports = function (middleware) { '^/templates/[\\w/]+.tpl', '^/api/login', '^/api/widgets/render', - '^/api/language/.+', + '^/public/language', '^/uploads/system/site-logo.png' ]; diff --git a/src/plugins.js b/src/plugins.js index 00a932a337..1a15dd560e 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -29,8 +29,6 @@ var middleware; Plugins.lessFiles = []; Plugins.clientScripts = []; Plugins.acpScripts = []; - Plugins.customLanguages = {}; - Plugins.customLanguageFallbacks = {}; Plugins.libraryPaths = []; Plugins.versionWarning = []; Plugins.languageCodes = []; diff --git a/src/plugins/load.js b/src/plugins/load.js index 0ccc095b7c..43c3d96f4c 100644 --- a/src/plugins/load.js +++ b/src/plugins/load.js @@ -9,8 +9,6 @@ var winston = require('winston'); var nconf = require('nconf'); var _ = require('underscore'); var file = require('../file'); - -var utils = require('../../public/src/utils'); var meta = require('../meta'); @@ -91,9 +89,6 @@ module.exports = function (Plugins) { function (next) { mapClientModules(pluginData, next); }, - function (next) { - loadLanguages(pluginData, next); - } ], function (err) { if (err) { winston.verbose('[plugins] Could not load plugin : ' + pluginData.id); @@ -252,60 +247,6 @@ module.exports = function (Plugins) { callback(); } - function loadLanguages(pluginData, callback) { - if (typeof pluginData.languages !== 'string') { - return callback(); - } - - var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); - var defaultLang = (pluginData.defaultLang || 'en_GB').replace('_', '-').replace('@', '-x-'); - - utils.walk(pathToFolder, function (err, languages) { - if (err) { - return callback(err); - } - - async.each(languages, function (pathToLang, next) { - fs.readFile(pathToLang, function (err, file) { - if (err) { - return next(err); - } - var data; - var language = path.dirname(pathToLang).split(/[\/\\]/).pop().replace('_', '-').replace('@', '-x-'); - var namespace = path.basename(pathToLang, '.json'); - var langNamespace = language + '/' + namespace; - - try { - data = JSON.parse(file.toString()); - } catch (err) { - winston.error('[plugins] Unable to parse custom language file: ' + pathToLang + '\r\n' + err.stack); - return next(err); - } - - Plugins.customLanguages[langNamespace] = Plugins.customLanguages[langNamespace] || {}; - Object.assign(Plugins.customLanguages[langNamespace], data); - - if (defaultLang && defaultLang === language) { - Plugins.languageCodes.filter(function (lang) { - return defaultLang !== lang; - }).forEach(function (lang) { - var langNS = lang + '/' + namespace; - Plugins.customLanguages[langNS] = Object.assign(Plugins.customLanguages[langNS] || {}, data); - }); - } - - next(); - }); - }, function (err) { - if (err) { - return callback(err); - } - - callback(); - }); - }); - } - function resolveModulePath(fullPath, relPath) { /** * With npm@3, dependencies can become flattened, and appear at the root level. @@ -363,6 +304,7 @@ module.exports = function (Plugins) { return callback(new Error('[[error:parse-error]]')); } + callback(null, pluginData); }); }; diff --git a/src/routes/index.js b/src/routes/index.js index d84c369fe1..99ff3a7f27 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -4,6 +4,7 @@ var nconf = require('nconf'); var winston = require('winston'); var path = require('path'); var async = require('async'); +var meta = require('../meta'); var controllers = require('../controllers'); var plugins = require('../plugins'); var user = require('../user'); @@ -145,7 +146,17 @@ module.exports = function (app, middleware, hotswapIds) { } app.use(middleware.privateUploads); - app.use(relativePath + '/api/language/:language/(([a-zA-Z0-9\\-_.\\/]+))', middleware.getTranslation); + app.use(relativePath + '/assets', express.static(path.join(__dirname, '../../', 'build/public'), { + maxAge: app.enabled('cache') ? 5184000000 : 0 + })); + + // DEPRECATED + app.use(relativePath + '/api/language', function (req, res) { + winston.warn('[deprecated] Accessing language files from `/api/language` is deprecated. ' + + 'Use `/assets/language/[langCode]/[namespace].json` for prefetch paths.'); + res.redirect(relativePath + '/assets/language' + req.path + '.json?' + meta.config['cache-buster']); + }); + app.use(relativePath, express.static(path.join(__dirname, '../../', 'public'), { maxAge: app.enabled('cache') ? 5184000000 : 0 }));