From b0645cc67db6dd50f2a27548613e195eb502374e Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Wed, 17 May 2017 17:25:41 -0600 Subject: [PATCH 1/6] Plugin load refactor --- src/meta/languages.js | 19 +-- src/plugins.js | 39 +----- src/plugins/data.js | 300 +++++++++++++++++++++++++++++++++++++++ src/plugins/load.js | 317 +++++++++--------------------------------- 4 files changed, 374 insertions(+), 301 deletions(-) create mode 100644 src/plugins/data.js diff --git a/src/meta/languages.js b/src/meta/languages.js index 90c3603677..28fea3c794 100644 --- a/src/meta/languages.js +++ b/src/meta/languages.js @@ -9,29 +9,14 @@ var rimraf = require('rimraf'); var file = require('../file'); var Plugins = require('../plugins'); -var db = require('../database'); var buildLanguagesPath = path.join(__dirname, '../../build/public/language'); -var coreLanguagesPath = path.join(__dirname, '../../public/language'); +var coreLanguagesPath = path.join(__dirname, '../../public/language'); function getTranslationTree(callback) { async.waterfall([ // get plugin data - function (next) { - db.getSortedSetRange('plugins:active', 0, -1, next); - }, - function (plugins, next) { - var pluginBasePath = path.join(__dirname, '../../node_modules'); - var paths = plugins.map(function (plugin) { - return path.join(pluginBasePath, plugin); - }); - - // Filter out plugins with invalid paths - async.filter(paths, file.exists, next); - }, - function (paths, next) { - async.map(paths, Plugins.loadPluginInfo, next); - }, + Plugins.data.getActive, // generate list of languages and namespaces function (plugins, next) { diff --git a/src/plugins.js b/src/plugins.js index 6c69553a05..d32a133648 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -8,10 +8,8 @@ var semver = require('semver'); var express = require('express'); var nconf = require('nconf'); -var db = require('./database'); var hotswap = require('./hotswap'); var file = require('./file'); -var languages = require('./languages'); var app; var middleware; @@ -20,6 +18,10 @@ var middleware; require('./plugins/install')(Plugins); require('./plugins/load')(Plugins); require('./plugins/hooks')(Plugins); + Plugins.data = require('./plugins/data'); + + Plugins.getPluginPaths = Plugins.data.getPluginPaths; + Plugins.loadPluginInfo = Plugins.data.loadPluginInfo; Plugins.libraries = {}; Plugins.loadedHooks = {}; @@ -30,7 +32,6 @@ var middleware; Plugins.acpScripts = []; Plugins.libraryPaths = []; Plugins.versionWarning = []; - Plugins.languageCodes = []; Plugins.soundpacks = []; Plugins.initialized = false; @@ -84,21 +85,7 @@ var middleware; Plugins.libraryPaths.length = 0; async.waterfall([ - function (next) { - // Build language code list - languages.list(function (err, languages) { - if (err) { - return next(err); - } - - Plugins.languageCodes = languages.map(function (data) { - return data.code; - }); - - next(); - }); - }, - async.apply(Plugins.getPluginPaths), + Plugins.getPluginPaths, function (paths, next) { async.eachSeries(paths, Plugins.loadPlugin, next); }, @@ -150,21 +137,7 @@ var middleware; var templates = {}; var tplName; - async.waterfall([ - async.apply(db.getSortedSetRange, 'plugins:active', 0, -1), - function (plugins, next) { - var pluginBasePath = path.join(__dirname, '../node_modules'); - var paths = plugins.map(function (plugin) { - return path.join(pluginBasePath, plugin); - }); - - // Filter out plugins with invalid paths - async.filter(paths, file.exists, next); - }, - function (paths, next) { - async.map(paths, Plugins.loadPluginInfo, next); - }, - ], function (err, plugins) { + Plugins.data.getActive(function (err, plugins) { if (err) { return callback(err); } diff --git a/src/plugins/data.js b/src/plugins/data.js new file mode 100644 index 0000000000..d2f1278c9d --- /dev/null +++ b/src/plugins/data.js @@ -0,0 +1,300 @@ +'use strict'; + +var fs = require('fs'); +var path = require('path'); +var async = require('async'); +var winston = require('winston'); + +var db = require('../database'); +var file = require('../file'); + +var Data = module.exports; + +var basePath = path.join(__dirname, '../../'); + +function getPluginPaths(callback) { + async.waterfall([ + function (next) { + db.getSortedSetRange('plugins:active', 0, -1, next); + }, + function (plugins, next) { + if (!Array.isArray(plugins)) { + return next(); + } + + plugins = plugins.filter(function (plugin) { + return plugin && typeof plugin === 'string'; + }).map(function (plugin) { + return path.join(__dirname, '../../node_modules/', plugin); + }); + + async.filter(plugins, file.exists, next); + }, + ], callback); +} +Data.getPluginPaths = getPluginPaths; + +function loadPluginInfo(pluginPath, callback) { + async.parallel({ + package: function (next) { + fs.readFile(path.join(pluginPath, 'package.json'), next); + }, + plugin: function (next) { + fs.readFile(path.join(pluginPath, 'plugin.json'), next); + }, + }, function (err, results) { + if (err) { + return callback(err); + } + var pluginData; + var packageData; + try { + pluginData = JSON.parse(results.plugin); + packageData = JSON.parse(results.package); + + pluginData.id = packageData.name; + pluginData.name = packageData.name; + pluginData.description = packageData.description; + pluginData.version = packageData.version; + pluginData.repository = packageData.repository; + pluginData.nbbpm = packageData.nbbpm; + pluginData.path = pluginPath; + } catch (err) { + var pluginDir = path.basename(pluginPath); + + winston.error('[plugins/' + pluginDir + '] Error in plugin.json or package.json! ' + err.message); + return callback(new Error('[[error:parse-error]]')); + } + + callback(null, pluginData); + }); +} +Data.loadPluginInfo = loadPluginInfo; + +function getAllPluginData(callback) { + async.waterfall([ + function (next) { + getPluginPaths(next); + }, + function (pluginPaths, next) { + async.map(pluginPaths, loadPluginInfo, next); + }, + ], callback); +} +Data.getActive = getAllPluginData; + +function getStaticDirectories(pluginData, callback) { + var validMappedPath = /^[\w\-_]+$/; + + if (!pluginData.staticDirs) { + return callback(); + } + + var dirs = Object.keys(pluginData.staticDirs); + if (!dirs.length) { + return callback(); + } + + var staticDirs = {}; + + async.each(dirs, function (route, next) { + if (!validMappedPath.test(route)) { + winston.warn('[plugins/' + pluginData.id + '] Invalid mapped path specified: ' + + route + '. Path must adhere to: ' + validMappedPath.toString()); + return next(); + } + + var dirPath = path.join(pluginData.path, pluginData.staticDirs[route]); + fs.stat(dirPath, function (err, stats) { + if (err && err.code === 'ENOENT') { + winston.warn('[plugins/' + pluginData.id + '] Mapped path \'' + + route + ' => ' + dirPath + '\' not found.'); + return next(); + } + if (err) { + return next(err); + } + + if (!stats.isDirectory()) { + winston.warn('[plugins/' + pluginData.id + '] Mapped path \'' + + route + ' => ' + dirPath + '\' is not a directory.'); + return next(); + } + + staticDirs[pluginData.id + '/' + route] = dirPath; + next(); + }); + }, function (err) { + callback(err, staticDirs); + }); +} +Data.getStaticDirectories = getStaticDirectories; + +function getFiles(pluginData, type, callback) { + if (!Array.isArray(pluginData[type]) || !pluginData[type].length) { + return callback(); + } + + if (global.env === 'development') { + winston.verbose('[plugins] Found ' + pluginData[type].length + ' ' + type + ' file(s) for plugin ' + pluginData.id); + } + + var files = pluginData[type].map(function (file) { + return path.join(pluginData.id, file); + }); + + callback(null, files); +} +Data.getFiles = getFiles; + +/** + * With npm@3, dependencies can become flattened, and appear at the root level. + * This method resolves these differences if it can. + */ +function resolveModulePath(basePath, modulePath, callback) { + var isNodeModule = /node_modules/; + + var currentPath = path.join(basePath, modulePath); + file.exists(currentPath, function (err, exists) { + if (err) { + return callback(err); + } + if (exists) { + return callback(null, currentPath); + } + if (!isNodeModule.test(modulePath)) { + winston.warn('[plugins] File not found: ' + currentPath + ' (Ignoring)'); + return callback(); + } + + var dirPath = path.dirname(basePath); + if (dirPath === basePath) { + winston.warn('[plugins] File not found: ' + currentPath + ' (Ignoring)'); + return callback(); + } + + resolveModulePath(dirPath, modulePath, callback); + }); +} + +function getScripts(pluginData, target, callback) { + target = (target === 'client') ? 'scripts' : 'acpScripts'; + + var input = pluginData[target]; + if (!Array.isArray(input) || !input.length) { + return callback(); + } + + var scripts = []; + async.each(input, function (filePath, next) { + resolveModulePath(pluginData.path, filePath, function (err, modulePath) { + if (err) { + return next(err); + } + + if (modulePath) { + scripts.push(modulePath); + } + next(); + }); + }, function (err) { + if (err) { + return callback(err); + } + + if (scripts.length && global.env === 'development') { + winston.verbose('[plugins] Found ' + scripts.length + ' js file(s) for plugin ' + pluginData.id); + } + callback(err, scripts); + }); +} +Data.getScripts = getScripts; + +function getModules(pluginData, callback) { + if (!pluginData.modules || !pluginData.hasOwnProperty('modules')) { + return callback(); + } + + var pluginModules = pluginData.modules; + + if (Array.isArray(pluginModules)) { + var strip = parseInt(pluginData.modulesStrip, 10) || 0; + + pluginModules = pluginModules.reduce(function (prev, modulePath) { + var key; + if (strip) { + key = modulePath.replace(new RegExp('.?(/[^/]+){' + strip + '}/'), ''); + } else { + key = path.basename(modulePath); + } + + prev[key] = modulePath; + return prev; + }, {}); + } + + var modules = {}; + async.each(Object.keys(pluginModules), function (key, next) { + resolveModulePath(pluginData.path, pluginModules[key], function (err, modulePath) { + if (err) { + return next(err); + } + + if (modulePath) { + modules[key] = path.relative(basePath, modulePath); + } + next(); + }); + }, function (err) { + if (err) { + return callback(err); + } + + if (global.env === 'development') { + var len = Object.keys(modules).length; + winston.verbose('[plugins] Found ' + len + ' AMD-style module(s) for plugin ' + pluginData.id); + } + callback(null, modules); + }); +} +Data.getModules = getModules; + +function getSoundpack(pluginData, callback) { + var spack = pluginData.soundpack; + if (!spack || !spack.dir || !spack.sounds) { + return callback(); + } + + var soundpack = {}; + soundpack.name = spack.name || pluginData.name; + soundpack.id = pluginData.id; + soundpack.dir = path.join(pluginData.path, spack.dir); + soundpack.sounds = {}; + + async.each(Object.keys(spack.sounds), function (name, next) { + var soundFile = spack.sounds[name]; + file.exists(path.join(soundpack.dir, soundFile), function (err, exists) { + if (err) { + return next(err); + } + if (!exists) { + winston.warn('[plugins] Sound file not found: ' + soundFile); + return next(); + } + + soundpack.sounds[name] = soundFile; + next(); + }); + }, function (err) { + if (err) { + return callback(err); + } + + if (global.env === 'development') { + var len = Object.keys(soundpack).length; + winston.verbose('[plugins] Found ' + len + ' sound file(s) for plugin ' + pluginData.id); + } + callback(null, soundpack); + }); +} +Data.getSoundpack = getSoundpack; diff --git a/src/plugins/load.js b/src/plugins/load.js index 04b65e9163..b939769588 100644 --- a/src/plugins/load.js +++ b/src/plugins/load.js @@ -1,38 +1,71 @@ 'use strict'; -var db = require('../database'); -var fs = require('fs'); var path = require('path'); var semver = require('semver'); var async = require('async'); var winston = require('winston'); var nconf = require('nconf'); -var _ = require('underscore'); -var file = require('../file'); -var meta = require('../meta'); +var meta = require('../meta'); module.exports = function (Plugins) { - Plugins.getPluginPaths = function (callback) { - async.waterfall([ - function (next) { - db.getSortedSetRange('plugins:active', 0, -1, next); + function registerPluginAssets(pluginData, fields, callback) { + function add(dest, arr) { + dest.push.apply(dest, arr || []); + } + + var handlers = { + staticDirs: function (next) { + Plugins.data.getStaticDirectories(pluginData, next); }, - function (plugins, next) { - if (!Array.isArray(plugins)) { - return next(); - } + cssFiles: function (next) { + Plugins.data.getFiles(pluginData, 'css', next); + }, + lessFiles: function (next) { + Plugins.data.getFiles(pluginData, 'less', next); + }, + clientScripts: function (next) { + Plugins.data.getScripts(pluginData, 'client', next); + }, + acpScripts: function (next) { + Plugins.data.getScripts(pluginData, 'acp', next); + }, + modules: function (next) { + Plugins.data.getModules(pluginData, next); + }, + soundpack: function (next) { + Plugins.data.getSoundpack(pluginData, next); + }, + }; + + var methods; + if (Array.isArray(fields)) { + methods = fields.reduce(function (prev, field) { + prev[field] = handlers[field]; + return prev; + }, {}); + } else { + methods = handlers; + } - plugins = plugins.filter(function (plugin) { - return plugin && typeof plugin === 'string'; - }).map(function (plugin) { - return path.join(__dirname, '../../node_modules/', plugin); - }); + async.parallel(methods, function (err, results) { + if (err) { + return callback(err); + } - async.filter(plugins, file.exists, next); - }, - ], callback); - }; + Object.assign(Plugins.staticDirs, results.staticDirs || {}); + add(Plugins.cssFiles, results.cssFiles); + add(Plugins.lessFiles, results.lessFiles); + add(Plugins.clientScripts, results.clientScripts); + add(Plugins.acpScripts, results.acpScripts); + Object.assign(meta.js.scripts.modules, results.modules || {}); + if (results.soundpack) { + Plugins.soundpacks.push(results.soundpack); + } + + callback(); + }); + } Plugins.prepareForBuild = function (callback) { Plugins.cssFiles.length = 0; @@ -42,29 +75,18 @@ module.exports = function (Plugins) { Plugins.soundpacks.length = 0; async.waterfall([ - async.apply(Plugins.getPluginPaths), - function (paths, next) { - async.map(paths, function (path, next) { - Plugins.loadPluginInfo(path, next); - }, next); - }, + Plugins.data.getActive, function (plugins, next) { async.each(plugins, function (pluginData, next) { - async.parallel([ - async.apply(mapFiles, pluginData, 'css', 'cssFiles'), - async.apply(mapFiles, pluginData, 'less', 'lessFiles'), - async.apply(mapClientSideScripts, pluginData), - async.apply(mapClientModules, pluginData), - async.apply(mapStaticDirectories, pluginData, pluginData.path), - async.apply(mapSoundpack, pluginData), - ], next); + // TODO: only load the data that's needed for the build + registerPluginAssets(pluginData, true, next); }, next); }, ], callback); }; Plugins.loadPlugin = function (pluginPath, callback) { - Plugins.loadPluginInfo(pluginPath, function (err, pluginData) { + Plugins.data.loadPluginInfo(pluginPath, function (err, pluginData) { if (err) { if (err.message === '[[error:parse-error]]') { return callback(); @@ -76,25 +98,13 @@ module.exports = function (Plugins) { async.parallel([ function (next) { - registerHooks(pluginData, pluginPath, next); - }, - function (next) { - mapStaticDirectories(pluginData, pluginPath, next); - }, - function (next) { - mapFiles(pluginData, 'css', 'cssFiles', next); - }, - function (next) { - mapFiles(pluginData, 'less', 'lessFiles', next); - }, - function (next) { - mapClientSideScripts(pluginData, next); + registerHooks(pluginData, next); }, function (next) { - mapClientModules(pluginData, next); - }, - function (next) { - mapSoundpack(pluginData, next); + // TODO: change this from `true` to `['soundpack']` + // this will skip several build-only plugin loading methods + // and only load soundpacks, which will speed up startup + registerPluginAssets(pluginData, true, next); }, ], function (err) { if (err) { @@ -124,12 +134,12 @@ module.exports = function (Plugins) { } } - function registerHooks(pluginData, pluginPath, callback) { + function registerHooks(pluginData, callback) { if (!pluginData.library) { return callback(); } - var libraryPath = path.join(pluginPath, pluginData.library); + var libraryPath = path.join(pluginData.path, pluginData.library); try { if (!Plugins.libraries[pluginData.id]) { @@ -149,199 +159,4 @@ module.exports = function (Plugins) { callback(); } } - - function mapStaticDirectories(pluginData, pluginPath, callback) { - var validMappedPath = /^[\w\-_]+$/; - - function mapStaticDirs(mappedPath, callback) { - if (Plugins.staticDirs[mappedPath]) { - winston.warn('[plugins/' + pluginData.id + '] Mapped path (' + mappedPath + ') already specified!'); - callback(); - } else if (!validMappedPath.test(mappedPath)) { - winston.warn('[plugins/' + pluginData.id + '] Invalid mapped path specified: ' + mappedPath + '. Path must adhere to: ' + validMappedPath.toString()); - callback(); - } else { - var realPath = pluginData.staticDirs[mappedPath]; - var staticDir = path.join(pluginPath, realPath); - - file.exists(staticDir, function (err, exists) { - if (exists) { - Plugins.staticDirs[pluginData.id + '/' + mappedPath] = staticDir; - } else { - winston.warn('[plugins/' + pluginData.id + '] Mapped path \'' + mappedPath + ' => ' + staticDir + '\' not found.'); - } - callback(err); - }); - } - } - - pluginData.staticDirs = pluginData.staticDirs || {}; - - var dirs = Object.keys(pluginData.staticDirs); - async.each(dirs, mapStaticDirs, callback); - } - - function mapFiles(pluginData, type, globalArray, callback) { - if (Array.isArray(pluginData[type])) { - if (global.env === 'development') { - winston.verbose('[plugins] Found ' + pluginData[type].length + ' ' + type + ' file(s) for plugin ' + pluginData.id); - } - - Plugins[globalArray] = Plugins[globalArray].concat(pluginData[type].map(function (file) { - return path.join(pluginData.id, file); - })); - } - callback(); - } - - function mapClientSideScripts(pluginData, callback) { - function mapScripts(scripts, param) { - if (Array.isArray(scripts) && scripts.length) { - if (global.env === 'development') { - winston.verbose('[plugins] Found ' + scripts.length + ' js file(s) for plugin ' + pluginData.id); - } - - Plugins[param] = Plugins[param].concat(scripts.map(function (file) { - return resolveModulePath(path.join(__dirname, '../../node_modules/', pluginData.id, file), file); - })).filter(Boolean); - } - } - mapScripts(pluginData.scripts, 'clientScripts'); - mapScripts(pluginData.acpScripts, 'acpScripts'); - - callback(); - } - - function mapClientModules(pluginData, callback) { - if (!pluginData.hasOwnProperty('modules')) { - return callback(); - } - - var modules = {}; - - if (Array.isArray(pluginData.modules)) { - if (global.env === 'development') { - winston.verbose('[plugins] Found ' + pluginData.modules.length + ' AMD-style module(s) for plugin ' + pluginData.id); - } - - var strip = pluginData.hasOwnProperty('modulesStrip') ? parseInt(pluginData.modulesStrip, 10) : 0; - - pluginData.modules.forEach(function (file) { - if (strip) { - modules[file.replace(new RegExp('.?(/[^/]+){' + strip + '}/'), '')] = path.join('./node_modules/', pluginData.id, file); - } else { - modules[path.basename(file)] = path.join('./node_modules/', pluginData.id, file); - } - }); - - meta.js.scripts.modules = _.extend(meta.js.scripts.modules, modules); - } else { - var keys = Object.keys(pluginData.modules); - - if (global.env === 'development') { - winston.verbose('[plugins] Found ' + keys.length + ' AMD-style module(s) for plugin ' + pluginData.id); - } - - for (var name in pluginData.modules) { - if (pluginData.modules.hasOwnProperty(name)) { - modules[name] = path.join('./node_modules/', pluginData.id, pluginData.modules[name]); - } - } - - meta.js.scripts.modules = _.extend(meta.js.scripts.modules, modules); - } - - callback(); - } - - function mapSoundpack(pluginData, callback) { - var soundpack = pluginData.soundpack; - if (!soundpack || !soundpack.dir || !soundpack.sounds) { - return callback(); - } - soundpack.name = soundpack.name || pluginData.name; - soundpack.id = pluginData.id; - soundpack.dir = path.join(pluginData.path, soundpack.dir); - async.each(Object.keys(soundpack.sounds), function (key, next) { - file.exists(path.join(soundpack.dir, soundpack.sounds[key]), function (err, exists) { - if (!exists) { - delete soundpack.sounds[key]; - } - - next(err); - }); - }, function (err) { - if (err) { - return callback(err); - } - - if (Object.keys(soundpack.sounds).length) { - Plugins.soundpacks.push(soundpack); - } - - callback(); - }); - } - - function resolveModulePath(fullPath, relPath) { - /** - * With npm@3, dependencies can become flattened, and appear at the root level. - * This method resolves these differences if it can. - */ - var matches = fullPath.match(/node_modules/g); - var atRootLevel = !matches || matches.length === 1; - - try { - fs.statSync(fullPath); - winston.verbose('[plugins/load] File found: ' + fullPath); - return fullPath; - } catch (e) { - // File not visible to the calling process, ascend to root level if possible and try again - if (!atRootLevel && relPath) { - winston.verbose('[plugins/load] File not found: ' + fullPath + ' (Ascending)'); - return resolveModulePath(path.join(__dirname, '../..', relPath)); - } - // Already at root level, file was simply not found - winston.warn('[plugins/load] File not found: ' + fullPath + ' (Ignoring)'); - return null; - } - } - - Plugins.loadPluginInfo = function (pluginPath, callback) { - async.parallel({ - package: function (next) { - fs.readFile(path.join(pluginPath, 'package.json'), next); - }, - plugin: function (next) { - fs.readFile(path.join(pluginPath, 'plugin.json'), next); - }, - }, function (err, results) { - if (err) { - return callback(err); - } - var pluginData; - var packageData; - try { - pluginData = JSON.parse(results.plugin); - packageData = JSON.parse(results.package); - - pluginData.id = packageData.name; - pluginData.name = packageData.name; - pluginData.description = packageData.description; - pluginData.version = packageData.version; - pluginData.repository = packageData.repository; - pluginData.nbbpm = packageData.nbbpm; - pluginData.path = pluginPath; - } catch (err) { - var pluginDir = pluginPath.split(path.sep); - pluginDir = pluginDir[pluginDir.length - 1]; - - winston.error('[plugins/' + pluginDir + '] Error in plugin.json or package.json! ' + err.message); - - return callback(new Error('[[error:parse-error]]')); - } - - callback(null, pluginData); - }); - }; }; From 90da6d889d92c9e2e1400a24d5d09f5af0b55d61 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Thu, 18 May 2017 01:32:20 -0600 Subject: [PATCH 2/6] Only load necessary plugin data Fix tests to work in this case Add more verbose messages to plugins/data --- src/meta/build.js | 12 ++++++-- src/plugins/data.js | 27 ++++++++-------- src/plugins/load.js | 31 ++++++++++++++----- test/build.js | 8 +++++ test/mocks/databasemock.js | 63 ++++++++++++++++++++++---------------- test/plugins.js | 1 - 6 files changed, 92 insertions(+), 50 deletions(-) diff --git a/src/meta/build.js b/src/meta/build.js index e88bbb17e9..dfeb4d7825 100644 --- a/src/meta/build.js +++ b/src/meta/build.js @@ -93,7 +93,7 @@ aliases = Object.keys(aliases).reduce(function (prev, key) { return prev; }, {}); -function beforeBuild(callback) { +function beforeBuild(targets, callback) { var db = require('../database'); var plugins = require('../plugins'); meta = require('../meta'); @@ -101,7 +101,9 @@ function beforeBuild(callback) { async.series([ db.init, meta.themes.setupPaths, - plugins.prepareForBuild, + function (next) { + plugins.prepareForBuild(targets, next); + }, ], function (err) { if (err) { winston.error('[build] Encountered error preparing for build: ' + err.message); @@ -160,6 +162,8 @@ function build(targets, callback) { return arr.indexOf(target) === i; }); + winston.verbose('[build] building the following targets: ' + targets.join(', ')); + if (typeof callback !== 'function') { callback = function (err) { if (err) { @@ -179,7 +183,9 @@ function build(targets, callback) { var startTime; var totalTime; async.series([ - beforeBuild, + function (next) { + beforeBuild(targets, next); + }, function (next) { var parallel = os.cpus().length > 1 && !nconf.get('series'); if (parallel) { diff --git a/src/plugins/data.js b/src/plugins/data.js index d2f1278c9d..965ad07bfc 100644 --- a/src/plugins/data.js +++ b/src/plugins/data.js @@ -125,7 +125,12 @@ function getStaticDirectories(pluginData, callback) { next(); }); }, function (err) { - callback(err, staticDirs); + if (err) { + return callback(err); + } + winston.verbose('[plugins] found ' + Object.keys(staticDirs).length + + ' static directories for ' + pluginData.id); + callback(null, staticDirs); }); } Data.getStaticDirectories = getStaticDirectories; @@ -135,9 +140,7 @@ function getFiles(pluginData, type, callback) { return callback(); } - if (global.env === 'development') { - winston.verbose('[plugins] Found ' + pluginData[type].length + ' ' + type + ' file(s) for plugin ' + pluginData.id); - } + winston.verbose('[plugins] Found ' + pluginData[type].length + ' ' + type + ' file(s) for plugin ' + pluginData.id); var files = pluginData[type].map(function (file) { return path.join(pluginData.id, file); @@ -202,7 +205,7 @@ function getScripts(pluginData, target, callback) { return callback(err); } - if (scripts.length && global.env === 'development') { + if (scripts.length) { winston.verbose('[plugins] Found ' + scripts.length + ' js file(s) for plugin ' + pluginData.id); } callback(err, scripts); @@ -250,10 +253,9 @@ function getModules(pluginData, callback) { return callback(err); } - if (global.env === 'development') { - var len = Object.keys(modules).length; - winston.verbose('[plugins] Found ' + len + ' AMD-style module(s) for plugin ' + pluginData.id); - } + var len = Object.keys(modules).length; + winston.verbose('[plugins] Found ' + len + ' AMD-style module(s) for plugin ' + pluginData.id); + callback(null, modules); }); } @@ -290,10 +292,9 @@ function getSoundpack(pluginData, callback) { return callback(err); } - if (global.env === 'development') { - var len = Object.keys(soundpack).length; - winston.verbose('[plugins] Found ' + len + ' sound file(s) for plugin ' + pluginData.id); - } + var len = Object.keys(soundpack.sounds).length; + winston.verbose('[plugins] Found ' + len + ' sound file(s) for plugin ' + pluginData.id); + callback(null, soundpack); }); } diff --git a/src/plugins/load.js b/src/plugins/load.js index b939769588..b0575670cf 100644 --- a/src/plugins/load.js +++ b/src/plugins/load.js @@ -67,19 +67,39 @@ module.exports = function (Plugins) { }); } - Plugins.prepareForBuild = function (callback) { + Plugins.prepareForBuild = function (targets, callback) { Plugins.cssFiles.length = 0; Plugins.lessFiles.length = 0; Plugins.clientScripts.length = 0; Plugins.acpScripts.length = 0; Plugins.soundpacks.length = 0; + var map = { + 'plugin static dirs': ['staticDirs'], + 'requirejs modules': ['modules'], + 'client js bundle': ['clientScripts'], + 'admin js bundle': ['acpScripts'], + 'client side styles': ['cssFiles', 'lessFiles'], + 'admin control panel styles': ['cssFiles', 'lessFiles'], + sounds: ['soundpack'], + }; + + var fields = targets.reduce(function (prev, target) { + if (!map[target]) { + return prev; + } + return prev.concat(map[target]); + }, []).filter(function (field, i, arr) { + return arr.indexOf(field) === i; + }); + + winston.verbose('[plugins] loading the following fields from plugin data: ' + fields.join(', ')); + async.waterfall([ Plugins.data.getActive, function (plugins, next) { async.each(plugins, function (pluginData, next) { - // TODO: only load the data that's needed for the build - registerPluginAssets(pluginData, true, next); + registerPluginAssets(pluginData, fields, next); }, next); }, ], callback); @@ -101,10 +121,7 @@ module.exports = function (Plugins) { registerHooks(pluginData, next); }, function (next) { - // TODO: change this from `true` to `['soundpack']` - // this will skip several build-only plugin loading methods - // and only load soundpacks, which will speed up startup - registerPluginAssets(pluginData, true, next); + registerPluginAssets(pluginData, ['soundpack'], next); }, ], function (err) { if (err) { diff --git a/test/build.js b/test/build.js index 3b6890ebe6..1f73f5746b 100644 --- a/test/build.js +++ b/test/build.js @@ -5,6 +5,10 @@ var assert = require('assert'); var db = require('./mocks/databasemock'); describe('Build', function () { + before(function (done) { + db.setupMockDefaults(done); + }); + it('should build all assets', function (done) { this.timeout(50000); var build = require('../src/meta/build'); @@ -13,4 +17,8 @@ describe('Build', function () { done(); }); }); + + after(function (done) { + db.emptydb(done); + }); }); diff --git a/test/mocks/databasemock.js b/test/mocks/databasemock.js index 4432f944f1..ca280dda9e 100644 --- a/test/mocks/databasemock.js +++ b/test/mocks/databasemock.js @@ -83,42 +83,19 @@ before(function (done) { this.timeout(30000); - var meta; - async.waterfall([ + async.series([ function (next) { db.init(next); }, function (next) { - db.emptydb(next); - }, - function (next) { - winston.info('test_database flushed'); - meta = require('../../src/meta'); - setupDefaultConfigs(meta, next); - }, - function (next) { - meta.configs.init(next); + setupMockDefaults(next); }, function (next) { db.initSessionStore(next); }, function (next) { - meta.dependencies.check(next); - }, - function (next) { - meta.config.postDelay = 0; - meta.config.initialPostDelay = 0; - meta.config.newbiePostDelay = 0; + var meta = require('../../src/meta'); - enableDefaultPlugins(next); - }, - function (next) { - meta.themes.set({ - type: 'local', - id: 'nodebb-theme-persona', - }, next); - }, - function (next) { // nconf defaults, if not set in config if (!nconf.get('sessionKey')) { nconf.set('sessionKey', 'express.sid'); @@ -154,6 +131,40 @@ ], done); }); + function setupMockDefaults(callback) { + var meta = require('../../src/meta'); + + async.series([ + function (next) { + db.emptydb(next); + }, + function (next) { + winston.info('test_database flushed'); + setupDefaultConfigs(meta, next); + }, + function (next) { + meta.configs.init(next); + }, + function (next) { + meta.dependencies.check(next); + }, + function (next) { + meta.config.postDelay = 0; + meta.config.initialPostDelay = 0; + meta.config.newbiePostDelay = 0; + + enableDefaultPlugins(next); + }, + function (next) { + meta.themes.set({ + type: 'local', + id: 'nodebb-theme-persona', + }, next); + }, + ], callback); + } + db.setupMockDefaults = setupMockDefaults; + function setupDefaultConfigs(meta, next) { winston.info('Populating database with default configs, if not already set...\n'); diff --git a/test/plugins.js b/test/plugins.js index 97a9c1908b..caccbe2851 100644 --- a/test/plugins.js +++ b/test/plugins.js @@ -16,7 +16,6 @@ describe('Plugins', function () { assert.ifError(err); assert(plugins.libraries[pluginId]); assert(plugins.loadedHooks['static:app.load']); - assert(plugins.staticDirs['nodebb-plugin-markdown/js']); done(); }); From b8baac1aad570031741b9aa1fe39495476b937f1 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Tue, 23 May 2017 18:07:13 -0600 Subject: [PATCH 3/6] async.apply --- src/meta/build.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/meta/build.js b/src/meta/build.js index edb7d526ba..488d32e4fb 100644 --- a/src/meta/build.js +++ b/src/meta/build.js @@ -100,9 +100,7 @@ function beforeBuild(targets, callback) { async.series([ db.init, meta.themes.setupPaths, - function (next) { - plugins.prepareForBuild(targets, next); - }, + async.apply(plugins.prepareForBuild, targets), ], function (err) { if (err) { winston.error('[build] Encountered error preparing for build: ' + err.message); @@ -182,7 +180,9 @@ function build(targets, callback) { var startTime; var totalTime; async.series([ - async.apply(beforeBuild, targets), + function (next) { + beforeBuild(targets, next); + }, function (next) { var threads = parseInt(nconf.get('threads'), 10); if (threads) { From 3dd282b7b9497e0a52f70d2ed29515623bbd420f Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Tue, 23 May 2017 21:55:23 -0600 Subject: [PATCH 4/6] Load languages with other plugin data - Added more tests - Should speed up builds - Allows for incremental builds in the future --- src/meta/languages.js | 130 ++++++++++++++----------------------- src/plugins.js | 2 + src/plugins/data.js | 35 ++++++++++ src/plugins/load.js | 12 ++++ test/build.js | 25 +++++-- test/mocks/databasemock.js | 7 +- 6 files changed, 122 insertions(+), 89 deletions(-) diff --git a/src/meta/languages.js b/src/meta/languages.js index 28fea3c794..5692533756 100644 --- a/src/meta/languages.js +++ b/src/meta/languages.js @@ -6,6 +6,7 @@ var async = require('async'); var fs = require('fs'); var mkdirp = require('mkdirp'); var rimraf = require('rimraf'); +var _ = require('underscore'); var file = require('../file'); var Plugins = require('../plugins'); @@ -15,73 +16,34 @@ var coreLanguagesPath = path.join(__dirname, '../../public/language'); function getTranslationTree(callback) { async.waterfall([ - // get plugin data - Plugins.data.getActive, - // generate list of languages and namespaces - function (plugins, next) { + function (next) { + file.walk(coreLanguagesPath, next); + }, + function (paths, next) { var languages = []; var namespaces = []; - // pull languages and namespaces from paths - function extrude(languageDir, paths) { - paths.forEach(function (p) { - var rel = p.split(languageDir)[1].split(/[/\\]/).slice(1); - var language = rel.shift().replace('_', '-').replace('@', '-x-'); - var namespace = rel.join('/').replace(/\.json$/, ''); - - if (!language || !namespace) { - return; - } - - if (languages.indexOf(language) === -1) { - languages.push(language); - } - if (namespaces.indexOf(namespace) === -1) { - namespaces.push(namespace); - } - }); - } - - plugins = plugins.filter(function (pluginData) { - return (typeof pluginData.languages === 'string'); - }); - async.parallel([ - // get core languages and namespaces - function (nxt) { - file.walk(coreLanguagesPath, function (err, paths) { - if (err) { - return nxt(err); - } - - extrude(coreLanguagesPath, paths); - nxt(); - }); - }, - // get plugin languages and namespaces - function (nxt) { - async.each(plugins, function (pluginData, cb) { - var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); - file.walk(pathToFolder, function (err, paths) { - if (err) { - return cb(err); - } - - extrude(pathToFolder, paths); - cb(); - }); - }, nxt); - }, - ], function (err) { - if (err) { - return next(err); + paths.forEach(function (p) { + if (!p.endsWith('.json')) { + return; + } + + var rel = path.relative(coreLanguagesPath, p).split(/[/\\]/); + var language = rel.shift().replace('_', '-').replace('@', '-x-'); + var namespace = rel.join('/').replace(/\.json$/, ''); + + if (!language || !namespace) { + return; } - next(null, { - languages: languages, - namespaces: namespaces, - plugins: plugins, - }); + languages.push(language); + namespaces.push(namespace); + }); + + next(null, { + languages: _.union(languages, Plugins.languageData.languages).sort().filter(Boolean), + namespaces: _.union(namespaces, Plugins.languageData.namespaces).sort().filter(Boolean), }); }, @@ -94,8 +56,8 @@ function getTranslationTree(callback) { }, function (x, next) { fs.writeFile(path.join(buildLanguagesPath, 'metadata.json'), JSON.stringify({ - languages: ref.languages.sort(), - namespaces: ref.namespaces.sort(), + languages: ref.languages, + namespaces: ref.namespaces, }), next); }, function (next) { @@ -110,40 +72,42 @@ function getTranslationTree(callback) { function (ref, next) { var languages = ref.languages; var namespaces = ref.namespaces; - var plugins = ref.plugins; + var plugins = _.values(Plugins.pluginsData).filter(function (plugin) { + return typeof plugin.languages === 'string'; + }); var tree = {}; - async.eachLimit(languages, 10, function (lang, nxt) { - async.eachLimit(namespaces, 10, function (ns, cb) { + async.eachLimit(languages, 10, function (lang, next) { + async.eachLimit(namespaces, 10, function (namespace, next) { var translations = {}; async.series([ // core first - function (n) { - fs.readFile(path.join(coreLanguagesPath, lang, ns + '.json'), function (err, buffer) { + function (cb) { + fs.readFile(path.join(coreLanguagesPath, lang, namespace + '.json'), function (err, buffer) { if (err) { if (err.code === 'ENOENT') { - return n(); + return cb(); } - return n(err); + return cb(err); } try { Object.assign(translations, JSON.parse(buffer.toString())); - n(); + cb(); } catch (err) { - n(err); + cb(err); } }); }, - function (n) { + function (cb) { // for each plugin, fallback in this order: // 1. correct language string (en-GB) // 2. old language string (en_GB) // 3. corrected plugin defaultLang (en-US) // 4. old plugin defaultLang (en_US) - async.eachLimit(plugins, 10, function (pluginData, call) { + async.eachLimit(plugins, 10, function (pluginData, done) { var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); var defaultLang = pluginData.defaultLang || 'en-GB'; @@ -153,7 +117,7 @@ function getTranslationTree(callback) { defaultLang.replace('_', '-').replace('@', '-x-'), defaultLang.replace('-', '_').replace('-x-', '@'), ], function (language, next) { - fs.readFile(path.join(pluginLanguages, language, ns + '.json'), function (err, buffer) { + fs.readFile(path.join(pluginLanguages, language, namespace + '.json'), function (err, buffer) { if (err) { if (err.code === 'ENOENT') { return next(null, false); @@ -168,21 +132,21 @@ function getTranslationTree(callback) { next(err); } }); - }, call); + }, done); }, function (err) { if (err) { - return n(err); + return cb(err); } if (Object.keys(translations).length) { tree[lang] = tree[lang] || {}; - tree[lang][ns] = translations; + tree[lang][namespace] = translations; } - n(); + cb(); }); }, - ], cb); - }, nxt); + ], next); + }, next); }, function (err) { next(err, tree); }); @@ -193,9 +157,9 @@ function getTranslationTree(callback) { // write translation hashes from the generated tree to language files function writeLanguageFiles(tree, callback) { // iterate over languages and namespaces - async.eachLimit(Object.keys(tree), 10, function (language, cb) { + async.eachLimit(Object.keys(tree), 100, function (language, cb) { var namespaces = tree[language]; - async.eachLimit(Object.keys(namespaces), 100, function (namespace, next) { + async.eachLimit(Object.keys(namespaces), 10, function (namespace, next) { var translations = namespaces[namespace]; var filePath = path.join(buildLanguagesPath, language, namespace + '.json'); diff --git a/src/plugins.js b/src/plugins.js index d32a133648..e3752bbb0c 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -23,6 +23,7 @@ var middleware; Plugins.getPluginPaths = Plugins.data.getPluginPaths; Plugins.loadPluginInfo = Plugins.data.loadPluginInfo; + Plugins.pluginsData = {}; Plugins.libraries = {}; Plugins.loadedHooks = {}; Plugins.staticDirs = {}; @@ -33,6 +34,7 @@ var middleware; Plugins.libraryPaths = []; Plugins.versionWarning = []; Plugins.soundpacks = []; + Plugins.languageData = {}; Plugins.initialized = false; diff --git a/src/plugins/data.js b/src/plugins/data.js index 965ad07bfc..afb8d095ab 100644 --- a/src/plugins/data.js +++ b/src/plugins/data.js @@ -299,3 +299,38 @@ function getSoundpack(pluginData, callback) { }); } Data.getSoundpack = getSoundpack; + +function getLanguageData(pluginData, callback) { + if (typeof pluginData.languages !== 'string') { + return callback(); + } + + var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages); + file.walk(pathToFolder, function (err, paths) { + if (err) { + return callback(err); + } + + var namespaces = []; + var languages = []; + + paths.forEach(function (p) { + var rel = path.relative(pathToFolder, p).split(/[/\\]/); + var language = rel.shift().replace('_', '-').replace('@', '-x-'); + var namespace = rel.join('/').replace(/\.json$/, ''); + + if (!language || !namespace) { + return; + } + + languages.push(language); + namespaces.push(namespace); + }); + + callback(null, { + languages: languages, + namespaces: namespaces, + }); + }); +} +Data.getLanguageData = getLanguageData; diff --git a/src/plugins/load.js b/src/plugins/load.js index b0575670cf..c2456397fe 100644 --- a/src/plugins/load.js +++ b/src/plugins/load.js @@ -5,6 +5,7 @@ var semver = require('semver'); var async = require('async'); var winston = require('winston'); var nconf = require('nconf'); +var _ = require('underscore'); var meta = require('../meta'); @@ -36,6 +37,9 @@ module.exports = function (Plugins) { soundpack: function (next) { Plugins.data.getSoundpack(pluginData, next); }, + languageData: function (next) { + Plugins.data.getLanguageData(pluginData, next); + }, }; var methods; @@ -62,6 +66,11 @@ module.exports = function (Plugins) { if (results.soundpack) { Plugins.soundpacks.push(results.soundpack); } + if (results.languageData) { + Plugins.languageData.languages = _.union(Plugins.languageData.languages, results.languageData.languages); + Plugins.languageData.namespaces = _.union(Plugins.languageData.namespaces, results.languageData.namespaces); + } + Plugins.pluginsData[pluginData.id] = pluginData; callback(); }); @@ -73,6 +82,8 @@ module.exports = function (Plugins) { Plugins.clientScripts.length = 0; Plugins.acpScripts.length = 0; Plugins.soundpacks.length = 0; + Plugins.languageData.languages = []; + Plugins.languageData.namespaces = []; var map = { 'plugin static dirs': ['staticDirs'], @@ -82,6 +93,7 @@ module.exports = function (Plugins) { 'client side styles': ['cssFiles', 'lessFiles'], 'admin control panel styles': ['cssFiles', 'lessFiles'], sounds: ['soundpack'], + languages: ['languageData'], }; var fields = targets.reduce(function (prev, target) { diff --git a/test/build.js b/test/build.js index ceb33962be..3a86008a24 100644 --- a/test/build.js +++ b/test/build.js @@ -107,6 +107,7 @@ describe('Build', function (done) { async.parallel([ async.apply(rimraf, path.join(__dirname, '../build/public')), db.setupMockDefaults, + async.apply(db.activatePlugin, 'nodebb-plugin-markdown'), ], done); }); @@ -181,10 +182,17 @@ describe('Build', function (done) { it('should build languages', function (done) { build.build(['languages'], function (err) { assert.ifError(err); - var filename = path.join(__dirname, '../build/public/language/en-GB/global.json'); - assert(file.existsSync(filename)); - var global = fs.readFileSync(filename).toString(); + + var globalFile = path.join(__dirname, '../build/public/language/en-GB/global.json'); + assert(file.existsSync(globalFile)); + var global = fs.readFileSync(globalFile).toString(); assert.strictEqual(JSON.parse(global).home, 'Home'); + + var mdFile = path.join(__dirname, '../build/public/language/en-GB/markdown.json'); + assert(file.existsSync(mdFile)); + var md = fs.readFileSync(mdFile).toString(); + assert.strictEqual(JSON.parse(md).bold, 'bolded text'); + done(); }); }); @@ -192,8 +200,15 @@ describe('Build', function (done) { it('should build sounds', function (done) { build.build(['sounds'], function (err) { assert.ifError(err); - var filename = path.join(__dirname, '../build/public/sounds/fileMap.json'); - assert(file.existsSync(filename)); + + var mapFile = path.join(__dirname, '../build/public/sounds/fileMap.json'); + assert(file.existsSync(mapFile)); + var fileMap = JSON.parse(fs.readFileSync(mapFile)); + assert.strictEqual(fileMap['Default | Deedle-dum'], 'nodebb-plugin-soundpack-default/notification.mp3'); + + var deebleDumFile = path.join(__dirname, '../build/public/sounds/nodebb-plugin-soundpack-default/notification.mp3'); + assert(file.existsSync(deebleDumFile)); + done(); }); }); diff --git a/test/mocks/databasemock.js b/test/mocks/databasemock.js index f64c02942a..675aa9e866 100644 --- a/test/mocks/databasemock.js +++ b/test/mocks/databasemock.js @@ -178,9 +178,14 @@ function enableDefaultPlugins(callback) { var defaultEnabled = [ 'nodebb-plugin-dbsearch', + 'nodebb-plugin-soundpack-default', ]; winston.info('[install/enableDefaultPlugins] activating default plugins', defaultEnabled); - db.sortedSetAdd('plugins:active', [0], defaultEnabled, callback); + db.sortedSetAdd('plugins:active', Object.keys(defaultEnabled), defaultEnabled, callback); } + +db.activatePlugin = function (id, callback) { + db.sortedSetAdd('plugins:active', Date.now(), id, callback); +}; From 8ce35c3cc261554d45226d4e414abb4a7b9bf921 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Wed, 24 May 2017 16:55:23 -0600 Subject: [PATCH 5/6] async.apply -_- --- src/meta/build.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/meta/build.js b/src/meta/build.js index 488d32e4fb..29155d55c3 100644 --- a/src/meta/build.js +++ b/src/meta/build.js @@ -180,9 +180,7 @@ function build(targets, callback) { var startTime; var totalTime; async.series([ - function (next) { - beforeBuild(targets, next); - }, + async.apply(beforeBuild, targets), function (next) { var threads = parseInt(nconf.get('threads'), 10); if (threads) { From c85bf9202d2aa633157bd1eca68f6c5b8b2a1988 Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Wed, 24 May 2017 17:19:04 -0600 Subject: [PATCH 6/6] Fix tests sometimes failing --- test/build.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/build.js b/test/build.js index 3a86008a24..0d61283eac 100644 --- a/test/build.js +++ b/test/build.js @@ -104,7 +104,7 @@ describe('Build', function (done) { var build = require('../src/meta/build'); before(function (done) { - async.parallel([ + async.series([ async.apply(rimraf, path.join(__dirname, '../build/public')), db.setupMockDefaults, async.apply(db.activatePlugin, 'nodebb-plugin-markdown'), @@ -184,14 +184,14 @@ describe('Build', function (done) { assert.ifError(err); var globalFile = path.join(__dirname, '../build/public/language/en-GB/global.json'); - assert(file.existsSync(globalFile)); + assert(file.existsSync(globalFile), 'global.json exists'); var global = fs.readFileSync(globalFile).toString(); - assert.strictEqual(JSON.parse(global).home, 'Home'); + assert.strictEqual(JSON.parse(global).home, 'Home', 'global.json contains correct translations'); var mdFile = path.join(__dirname, '../build/public/language/en-GB/markdown.json'); - assert(file.existsSync(mdFile)); + assert(file.existsSync(mdFile), 'markdown.json exists'); var md = fs.readFileSync(mdFile).toString(); - assert.strictEqual(JSON.parse(md).bold, 'bolded text'); + assert.strictEqual(JSON.parse(md).bold, 'bolded text', 'markdown.json contains correct translations'); done(); });