From 3dd282b7b9497e0a52f70d2ed29515623bbd420f Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Tue, 23 May 2017 21:55:23 -0600 Subject: [PATCH] Load languages with other plugin data - Added more tests - Should speed up builds - Allows for incremental builds in the future --- src/meta/languages.js | 130 ++++++++++++++----------------------- src/plugins.js | 2 + src/plugins/data.js | 35 ++++++++++ src/plugins/load.js | 12 ++++ test/build.js | 25 +++++-- test/mocks/databasemock.js | 7 +- 6 files changed, 122 insertions(+), 89 deletions(-) diff --git a/src/meta/languages.js b/src/meta/languages.js index 28fea3c794..5692533756 100644 --- a/src/meta/languages.js +++ b/src/meta/languages.js @@ -6,6 +6,7 @@ var async = require('async'); var fs = require('fs'); var mkdirp = require('mkdirp'); var rimraf = require('rimraf'); +var _ = require('underscore'); var file = require('../file'); var Plugins = require('../plugins'); @@ -15,73 +16,34 @@ var coreLanguagesPath = path.join(__dirname, '../../public/language'); function getTranslationTree(callback) { async.waterfall([ - // get plugin data - Plugins.data.getActive, - // generate list of languages and namespaces - function (plugins, next) { + function (next) { + file.walk(coreLanguagesPath, next); + }, + function (paths, next) { var languages = []; var namespaces = []; - // pull languages and namespaces from paths - function extrude(languageDir, paths) { - paths.forEach(function (p) { - var rel = p.split(languageDir)[1].split(/[/\\]/).slice(1); - var language = rel.shift().replace('_', '-').replace('@', '-x-'); - var namespace = rel.join('/').replace(/\.json$/, ''); - - if (!language || !namespace) { - return; - } - - if (languages.indexOf(language) === -1) { - languages.push(language); - } - if (namespaces.indexOf(namespace) === -1) { - namespaces.push(namespace); - } - }); - } - - plugins = plugins.filter(function (pluginData) { - return (typeof pluginData.languages === 'string'); - }); - async.parallel([ - // get core languages and namespaces - function (nxt) { - file.walk(coreLanguagesPath, function (err, paths) { - if (err) { - return nxt(err); - } - - extrude(coreLanguagesPath, paths); - nxt(); - }); - }, - // get plugin languages and namespaces - function (nxt) { - async.each(plugins, function (pluginData, cb) { - var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); - file.walk(pathToFolder, function (err, paths) { - if (err) { - return cb(err); - } - - extrude(pathToFolder, paths); - cb(); - }); - }, nxt); - }, - ], function (err) { - if (err) { - return next(err); + paths.forEach(function (p) { + if (!p.endsWith('.json')) { + return; + } + + var rel = path.relative(coreLanguagesPath, p).split(/[/\\]/); + var language = rel.shift().replace('_', '-').replace('@', '-x-'); + var namespace = rel.join('/').replace(/\.json$/, ''); + + if (!language || !namespace) { + return; } - next(null, { - languages: languages, - namespaces: namespaces, - plugins: plugins, - }); + languages.push(language); + namespaces.push(namespace); + }); + + next(null, { + languages: _.union(languages, Plugins.languageData.languages).sort().filter(Boolean), + namespaces: _.union(namespaces, Plugins.languageData.namespaces).sort().filter(Boolean), }); }, @@ -94,8 +56,8 @@ function getTranslationTree(callback) { }, function (x, next) { fs.writeFile(path.join(buildLanguagesPath, 'metadata.json'), JSON.stringify({ - languages: ref.languages.sort(), - namespaces: ref.namespaces.sort(), + languages: ref.languages, + namespaces: ref.namespaces, }), next); }, function (next) { @@ -110,40 +72,42 @@ function getTranslationTree(callback) { function (ref, next) { var languages = ref.languages; var namespaces = ref.namespaces; - var plugins = ref.plugins; + var plugins = _.values(Plugins.pluginsData).filter(function (plugin) { + return typeof plugin.languages === 'string'; + }); var tree = {}; - async.eachLimit(languages, 10, function (lang, nxt) { - async.eachLimit(namespaces, 10, function (ns, cb) { + async.eachLimit(languages, 10, function (lang, next) { + async.eachLimit(namespaces, 10, function (namespace, next) { var translations = {}; async.series([ // core first - function (n) { - fs.readFile(path.join(coreLanguagesPath, lang, ns + '.json'), function (err, buffer) { + function (cb) { + fs.readFile(path.join(coreLanguagesPath, lang, namespace + '.json'), function (err, buffer) { if (err) { if (err.code === 'ENOENT') { - return n(); + return cb(); } - return n(err); + return cb(err); } try { Object.assign(translations, JSON.parse(buffer.toString())); - n(); + cb(); } catch (err) { - n(err); + cb(err); } }); }, - function (n) { + function (cb) { // for each plugin, fallback in this order: // 1. correct language string (en-GB) // 2. old language string (en_GB) // 3. corrected plugin defaultLang (en-US) // 4. old plugin defaultLang (en_US) - async.eachLimit(plugins, 10, function (pluginData, call) { + async.eachLimit(plugins, 10, function (pluginData, done) { var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); var defaultLang = pluginData.defaultLang || 'en-GB'; @@ -153,7 +117,7 @@ function getTranslationTree(callback) { defaultLang.replace('_', '-').replace('@', '-x-'), defaultLang.replace('-', '_').replace('-x-', '@'), ], function (language, next) { - fs.readFile(path.join(pluginLanguages, language, ns + '.json'), function (err, buffer) { + fs.readFile(path.join(pluginLanguages, language, namespace + '.json'), function (err, buffer) { if (err) { if (err.code === 'ENOENT') { return next(null, false); @@ -168,21 +132,21 @@ function getTranslationTree(callback) { next(err); } }); - }, call); + }, done); }, function (err) { if (err) { - return n(err); + return cb(err); } if (Object.keys(translations).length) { tree[lang] = tree[lang] || {}; - tree[lang][ns] = translations; + tree[lang][namespace] = translations; } - n(); + cb(); }); }, - ], cb); - }, nxt); + ], next); + }, next); }, function (err) { next(err, tree); }); @@ -193,9 +157,9 @@ function getTranslationTree(callback) { // write translation hashes from the generated tree to language files function writeLanguageFiles(tree, callback) { // iterate over languages and namespaces - async.eachLimit(Object.keys(tree), 10, function (language, cb) { + async.eachLimit(Object.keys(tree), 100, function (language, cb) { var namespaces = tree[language]; - async.eachLimit(Object.keys(namespaces), 100, function (namespace, next) { + async.eachLimit(Object.keys(namespaces), 10, function (namespace, next) { var translations = namespaces[namespace]; var filePath = path.join(buildLanguagesPath, language, namespace + '.json'); diff --git a/src/plugins.js b/src/plugins.js index d32a133648..e3752bbb0c 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -23,6 +23,7 @@ var middleware; Plugins.getPluginPaths = Plugins.data.getPluginPaths; Plugins.loadPluginInfo = Plugins.data.loadPluginInfo; + Plugins.pluginsData = {}; Plugins.libraries = {}; Plugins.loadedHooks = {}; Plugins.staticDirs = {}; @@ -33,6 +34,7 @@ var middleware; Plugins.libraryPaths = []; Plugins.versionWarning = []; Plugins.soundpacks = []; + Plugins.languageData = {}; Plugins.initialized = false; diff --git a/src/plugins/data.js b/src/plugins/data.js index 965ad07bfc..afb8d095ab 100644 --- a/src/plugins/data.js +++ b/src/plugins/data.js @@ -299,3 +299,38 @@ function getSoundpack(pluginData, callback) { }); } Data.getSoundpack = getSoundpack; + +function getLanguageData(pluginData, callback) { + if (typeof pluginData.languages !== 'string') { + return callback(); + } + + var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); + file.walk(pathToFolder, function (err, paths) { + if (err) { + return callback(err); + } + + var namespaces = []; + var languages = []; + + paths.forEach(function (p) { + var rel = path.relative(pathToFolder, p).split(/[/\\]/); + var language = rel.shift().replace('_', '-').replace('@', '-x-'); + var namespace = rel.join('/').replace(/\.json$/, ''); + + if (!language || !namespace) { + return; + } + + languages.push(language); + namespaces.push(namespace); + }); + + callback(null, { + languages: languages, + namespaces: namespaces, + }); + }); +} +Data.getLanguageData = getLanguageData; diff --git a/src/plugins/load.js b/src/plugins/load.js index b0575670cf..c2456397fe 100644 --- a/src/plugins/load.js +++ b/src/plugins/load.js @@ -5,6 +5,7 @@ var semver = require('semver'); var async = require('async'); var winston = require('winston'); var nconf = require('nconf'); +var _ = require('underscore'); var meta = require('../meta'); @@ -36,6 +37,9 @@ module.exports = function (Plugins) { soundpack: function (next) { Plugins.data.getSoundpack(pluginData, next); }, + languageData: function (next) { + Plugins.data.getLanguageData(pluginData, next); + }, }; var methods; @@ -62,6 +66,11 @@ module.exports = function (Plugins) { if (results.soundpack) { Plugins.soundpacks.push(results.soundpack); } + if (results.languageData) { + Plugins.languageData.languages = _.union(Plugins.languageData.languages, results.languageData.languages); + Plugins.languageData.namespaces = _.union(Plugins.languageData.namespaces, results.languageData.namespaces); + } + Plugins.pluginsData[pluginData.id] = pluginData; callback(); }); @@ -73,6 +82,8 @@ module.exports = function (Plugins) { Plugins.clientScripts.length = 0; Plugins.acpScripts.length = 0; Plugins.soundpacks.length = 0; + Plugins.languageData.languages = []; + Plugins.languageData.namespaces = []; var map = { 'plugin static dirs': ['staticDirs'], @@ -82,6 +93,7 @@ module.exports = function (Plugins) { 'client side styles': ['cssFiles', 'lessFiles'], 'admin control panel styles': ['cssFiles', 'lessFiles'], sounds: ['soundpack'], + languages: ['languageData'], }; var fields = targets.reduce(function (prev, target) { diff --git a/test/build.js b/test/build.js index ceb33962be..3a86008a24 100644 --- a/test/build.js +++ b/test/build.js @@ -107,6 +107,7 @@ describe('Build', function (done) { async.parallel([ async.apply(rimraf, path.join(__dirname, '../build/public')), db.setupMockDefaults, + async.apply(db.activatePlugin, 'nodebb-plugin-markdown'), ], done); }); @@ -181,10 +182,17 @@ describe('Build', function (done) { it('should build languages', function (done) { build.build(['languages'], function (err) { assert.ifError(err); - var filename = path.join(__dirname, '../build/public/language/en-GB/global.json'); - assert(file.existsSync(filename)); - var global = fs.readFileSync(filename).toString(); + + var globalFile = path.join(__dirname, '../build/public/language/en-GB/global.json'); + assert(file.existsSync(globalFile)); + var global = fs.readFileSync(globalFile).toString(); assert.strictEqual(JSON.parse(global).home, 'Home'); + + var mdFile = path.join(__dirname, '../build/public/language/en-GB/markdown.json'); + assert(file.existsSync(mdFile)); + var md = fs.readFileSync(mdFile).toString(); + assert.strictEqual(JSON.parse(md).bold, 'bolded text'); + done(); }); }); @@ -192,8 +200,15 @@ describe('Build', function (done) { it('should build sounds', function (done) { build.build(['sounds'], function (err) { assert.ifError(err); - var filename = path.join(__dirname, '../build/public/sounds/fileMap.json'); - assert(file.existsSync(filename)); + + var mapFile = path.join(__dirname, '../build/public/sounds/fileMap.json'); + assert(file.existsSync(mapFile)); + var fileMap = JSON.parse(fs.readFileSync(mapFile)); + assert.strictEqual(fileMap['Default | Deedle-dum'], 'nodebb-plugin-soundpack-default/notification.mp3'); + + var deebleDumFile = path.join(__dirname, '../build/public/sounds/nodebb-plugin-soundpack-default/notification.mp3'); + assert(file.existsSync(deebleDumFile)); + done(); }); }); diff --git a/test/mocks/databasemock.js b/test/mocks/databasemock.js index f64c02942a..675aa9e866 100644 --- a/test/mocks/databasemock.js +++ b/test/mocks/databasemock.js @@ -178,9 +178,14 @@ function enableDefaultPlugins(callback) { var defaultEnabled = [ 'nodebb-plugin-dbsearch', + 'nodebb-plugin-soundpack-default', ]; winston.info('[install/enableDefaultPlugins] activating default plugins', defaultEnabled); - db.sortedSetAdd('plugins:active', [0], defaultEnabled, callback); + db.sortedSetAdd('plugins:active', Object.keys(defaultEnabled), defaultEnabled, callback); } + +db.activatePlugin = function (id, callback) { + db.sortedSetAdd('plugins:active', Date.now(), id, callback); +};