From b0645cc67db6dd50f2a27548613e195eb502374e Mon Sep 17 00:00:00 2001 From: Peter Jaszkowiak Date: Wed, 17 May 2017 17:25:41 -0600 Subject: [PATCH] Plugin load refactor --- src/meta/languages.js | 19 +-- src/plugins.js | 39 +----- src/plugins/data.js | 300 +++++++++++++++++++++++++++++++++++++++ src/plugins/load.js | 317 +++++++++--------------------------------- 4 files changed, 374 insertions(+), 301 deletions(-) create mode 100644 src/plugins/data.js diff --git a/src/meta/languages.js b/src/meta/languages.js index 90c3603677..28fea3c794 100644 --- a/src/meta/languages.js +++ b/src/meta/languages.js @@ -9,29 +9,14 @@ var rimraf = require('rimraf'); var file = require('../file'); var Plugins = require('../plugins'); -var db = require('../database'); var buildLanguagesPath = path.join(__dirname, '../../build/public/language'); -var coreLanguagesPath = path.join(__dirname, '../../public/language'); +var coreLanguagesPath = path.join(__dirname, '../../public/language'); function getTranslationTree(callback) { async.waterfall([ // get plugin data - function (next) { - db.getSortedSetRange('plugins:active', 0, -1, next); - }, - function (plugins, next) { - var pluginBasePath = path.join(__dirname, '../../node_modules'); - var paths = plugins.map(function (plugin) { - return path.join(pluginBasePath, plugin); - }); - - // Filter out plugins with invalid paths - async.filter(paths, file.exists, next); - }, - function (paths, next) { - async.map(paths, Plugins.loadPluginInfo, next); - }, + Plugins.data.getActive, // generate list of languages and namespaces function (plugins, next) { diff --git a/src/plugins.js b/src/plugins.js index 6c69553a05..d32a133648 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -8,10 +8,8 @@ var semver = require('semver'); var express = require('express'); var nconf = require('nconf'); -var db = require('./database'); var hotswap = require('./hotswap'); var file = require('./file'); -var languages = require('./languages'); var app; var middleware; @@ -20,6 +18,10 @@ var middleware; require('./plugins/install')(Plugins); require('./plugins/load')(Plugins); require('./plugins/hooks')(Plugins); + Plugins.data = require('./plugins/data'); + + Plugins.getPluginPaths = Plugins.data.getPluginPaths; + Plugins.loadPluginInfo = Plugins.data.loadPluginInfo; Plugins.libraries = {}; Plugins.loadedHooks = {}; @@ -30,7 +32,6 @@ var middleware; Plugins.acpScripts = []; Plugins.libraryPaths = []; Plugins.versionWarning = []; - Plugins.languageCodes = []; Plugins.soundpacks = []; Plugins.initialized = false; @@ -84,21 +85,7 @@ var middleware; Plugins.libraryPaths.length = 0; async.waterfall([ - function (next) { - // Build language code list - languages.list(function (err, languages) { - if (err) { - return next(err); - } - - Plugins.languageCodes = languages.map(function (data) { - return data.code; - }); - - next(); - }); - }, - async.apply(Plugins.getPluginPaths), + Plugins.getPluginPaths, function (paths, next) { async.eachSeries(paths, Plugins.loadPlugin, next); }, @@ -150,21 +137,7 @@ var middleware; var templates = {}; var tplName; - async.waterfall([ - async.apply(db.getSortedSetRange, 'plugins:active', 0, -1), - function (plugins, next) { - var pluginBasePath = path.join(__dirname, '../node_modules'); - var paths = plugins.map(function (plugin) { - return path.join(pluginBasePath, plugin); - }); - - // Filter out plugins with invalid paths - async.filter(paths, file.exists, next); - }, - function (paths, next) { - async.map(paths, Plugins.loadPluginInfo, next); - }, - ], function (err, plugins) { + Plugins.data.getActive(function (err, plugins) { if (err) { return callback(err); } diff --git a/src/plugins/data.js b/src/plugins/data.js new file mode 100644 index 0000000000..d2f1278c9d --- /dev/null +++ b/src/plugins/data.js @@ -0,0 +1,300 @@ +'use strict'; + +var fs = require('fs'); +var path = require('path'); +var async = require('async'); +var winston = require('winston'); + +var db = require('../database'); +var file = require('../file'); + +var Data = module.exports; + +var basePath = path.join(__dirname, '../../'); + +function getPluginPaths(callback) { + async.waterfall([ + function (next) { + db.getSortedSetRange('plugins:active', 0, -1, next); + }, + function (plugins, next) { + if (!Array.isArray(plugins)) { + return next(); + } + + plugins = plugins.filter(function (plugin) { + return plugin && typeof plugin === 'string'; + }).map(function (plugin) { + return path.join(__dirname, '../../node_modules/', plugin); + }); + + async.filter(plugins, file.exists, next); + }, + ], callback); +} +Data.getPluginPaths = getPluginPaths; + +function loadPluginInfo(pluginPath, callback) { + async.parallel({ + package: function (next) { + fs.readFile(path.join(pluginPath, 'package.json'), next); + }, + plugin: function (next) { + fs.readFile(path.join(pluginPath, 'plugin.json'), next); + }, + }, function (err, results) { + if (err) { + return callback(err); + } + var pluginData; + var packageData; + try { + pluginData = JSON.parse(results.plugin); + packageData = JSON.parse(results.package); + + pluginData.id = packageData.name; + pluginData.name = packageData.name; + pluginData.description = packageData.description; + pluginData.version = packageData.version; + pluginData.repository = packageData.repository; + pluginData.nbbpm = packageData.nbbpm; + pluginData.path = pluginPath; + } catch (err) { + var pluginDir = path.basename(pluginPath); + + winston.error('[plugins/' + pluginDir + '] Error in plugin.json or package.json! ' + err.message); + return callback(new Error('[[error:parse-error]]')); + } + + callback(null, pluginData); + }); +} +Data.loadPluginInfo = loadPluginInfo; + +function getAllPluginData(callback) { + async.waterfall([ + function (next) { + getPluginPaths(next); + }, + function (pluginPaths, next) { + async.map(pluginPaths, loadPluginInfo, next); + }, + ], callback); +} +Data.getActive = getAllPluginData; + +function getStaticDirectories(pluginData, callback) { + var validMappedPath = /^[\w\-_]+$/; + + if (!pluginData.staticDirs) { + return callback(); + } + + var dirs = Object.keys(pluginData.staticDirs); + if (!dirs.length) { + return callback(); + } + + var staticDirs = {}; + + async.each(dirs, function (route, next) { + if (!validMappedPath.test(route)) { + winston.warn('[plugins/' + pluginData.id + '] Invalid mapped path specified: ' + + route + '. Path must adhere to: ' + validMappedPath.toString()); + return next(); + } + + var dirPath = path.join(pluginData.path, pluginData.staticDirs[route]); + fs.stat(dirPath, function (err, stats) { + if (err && err.code === 'ENOENT') { + winston.warn('[plugins/' + pluginData.id + '] Mapped path \'' + + route + ' => ' + dirPath + '\' not found.'); + return next(); + } + if (err) { + return next(err); + } + + if (!stats.isDirectory()) { + winston.warn('[plugins/' + pluginData.id + '] Mapped path \'' + + route + ' => ' + dirPath + '\' is not a directory.'); + return next(); + } + + staticDirs[pluginData.id + '/' + route] = dirPath; + next(); + }); + }, function (err) { + callback(err, staticDirs); + }); +} +Data.getStaticDirectories = getStaticDirectories; + +function getFiles(pluginData, type, callback) { + if (!Array.isArray(pluginData[type]) || !pluginData[type].length) { + return callback(); + } + + if (global.env === 'development') { + winston.verbose('[plugins] Found ' + pluginData[type].length + ' ' + type + ' file(s) for plugin ' + pluginData.id); + } + + var files = pluginData[type].map(function (file) { + return path.join(pluginData.id, file); + }); + + callback(null, files); +} +Data.getFiles = getFiles; + +/** + * With npm@3, dependencies can become flattened, and appear at the root level. + * This method resolves these differences if it can. + */ +function resolveModulePath(basePath, modulePath, callback) { + var isNodeModule = /node_modules/; + + var currentPath = path.join(basePath, modulePath); + file.exists(currentPath, function (err, exists) { + if (err) { + return callback(err); + } + if (exists) { + return callback(null, currentPath); + } + if (!isNodeModule.test(modulePath)) { + winston.warn('[plugins] File not found: ' + currentPath + ' (Ignoring)'); + return callback(); + } + + var dirPath = path.dirname(basePath); + if (dirPath === basePath) { + winston.warn('[plugins] File not found: ' + currentPath + ' (Ignoring)'); + return callback(); + } + + resolveModulePath(dirPath, modulePath, callback); + }); +} + +function getScripts(pluginData, target, callback) { + target = (target === 'client') ? 'scripts' : 'acpScripts'; + + var input = pluginData[target]; + if (!Array.isArray(input) || !input.length) { + return callback(); + } + + var scripts = []; + async.each(input, function (filePath, next) { + resolveModulePath(pluginData.path, filePath, function (err, modulePath) { + if (err) { + return next(err); + } + + if (modulePath) { + scripts.push(modulePath); + } + next(); + }); + }, function (err) { + if (err) { + return callback(err); + } + + if (scripts.length && global.env === 'development') { + winston.verbose('[plugins] Found ' + scripts.length + ' js file(s) for plugin ' + pluginData.id); + } + callback(err, scripts); + }); +} +Data.getScripts = getScripts; + +function getModules(pluginData, callback) { + if (!pluginData.modules || !pluginData.hasOwnProperty('modules')) { + return callback(); + } + + var pluginModules = pluginData.modules; + + if (Array.isArray(pluginModules)) { + var strip = parseInt(pluginData.modulesStrip, 10) || 0; + + pluginModules = pluginModules.reduce(function (prev, modulePath) { + var key; + if (strip) { + key = modulePath.replace(new RegExp('.?(/[^/]+){' + strip + '}/'), ''); + } else { + key = path.basename(modulePath); + } + + prev[key] = modulePath; + return prev; + }, {}); + } + + var modules = {}; + async.each(Object.keys(pluginModules), function (key, next) { + resolveModulePath(pluginData.path, pluginModules[key], function (err, modulePath) { + if (err) { + return next(err); + } + + if (modulePath) { + modules[key] = path.relative(basePath, modulePath); + } + next(); + }); + }, function (err) { + if (err) { + return callback(err); + } + + if (global.env === 'development') { + var len = Object.keys(modules).length; + winston.verbose('[plugins] Found ' + len + ' AMD-style module(s) for plugin ' + pluginData.id); + } + callback(null, modules); + }); +} +Data.getModules = getModules; + +function getSoundpack(pluginData, callback) { + var spack = pluginData.soundpack; + if (!spack || !spack.dir || !spack.sounds) { + return callback(); + } + + var soundpack = {}; + soundpack.name = spack.name || pluginData.name; + soundpack.id = pluginData.id; + soundpack.dir = path.join(pluginData.path, spack.dir); + soundpack.sounds = {}; + + async.each(Object.keys(spack.sounds), function (name, next) { + var soundFile = spack.sounds[name]; + file.exists(path.join(soundpack.dir, soundFile), function (err, exists) { + if (err) { + return next(err); + } + if (!exists) { + winston.warn('[plugins] Sound file not found: ' + soundFile); + return next(); + } + + soundpack.sounds[name] = soundFile; + next(); + }); + }, function (err) { + if (err) { + return callback(err); + } + + if (global.env === 'development') { + var len = Object.keys(soundpack).length; + winston.verbose('[plugins] Found ' + len + ' sound file(s) for plugin ' + pluginData.id); + } + callback(null, soundpack); + }); +} +Data.getSoundpack = getSoundpack; diff --git a/src/plugins/load.js b/src/plugins/load.js index 04b65e9163..b939769588 100644 --- a/src/plugins/load.js +++ b/src/plugins/load.js @@ -1,38 +1,71 @@ 'use strict'; -var db = require('../database'); -var fs = require('fs'); var path = require('path'); var semver = require('semver'); var async = require('async'); var winston = require('winston'); var nconf = require('nconf'); -var _ = require('underscore'); -var file = require('../file'); -var meta = require('../meta'); +var meta = require('../meta'); module.exports = function (Plugins) { - Plugins.getPluginPaths = function (callback) { - async.waterfall([ - function (next) { - db.getSortedSetRange('plugins:active', 0, -1, next); + function registerPluginAssets(pluginData, fields, callback) { + function add(dest, arr) { + dest.push.apply(dest, arr || []); + } + + var handlers = { + staticDirs: function (next) { + Plugins.data.getStaticDirectories(pluginData, next); }, - function (plugins, next) { - if (!Array.isArray(plugins)) { - return next(); - } + cssFiles: function (next) { + Plugins.data.getFiles(pluginData, 'css', next); + }, + lessFiles: function (next) { + Plugins.data.getFiles(pluginData, 'less', next); + }, + clientScripts: function (next) { + Plugins.data.getScripts(pluginData, 'client', next); + }, + acpScripts: function (next) { + Plugins.data.getScripts(pluginData, 'acp', next); + }, + modules: function (next) { + Plugins.data.getModules(pluginData, next); + }, + soundpack: function (next) { + Plugins.data.getSoundpack(pluginData, next); + }, + }; + + var methods; + if (Array.isArray(fields)) { + methods = fields.reduce(function (prev, field) { + prev[field] = handlers[field]; + return prev; + }, {}); + } else { + methods = handlers; + } - plugins = plugins.filter(function (plugin) { - return plugin && typeof plugin === 'string'; - }).map(function (plugin) { - return path.join(__dirname, '../../node_modules/', plugin); - }); + async.parallel(methods, function (err, results) { + if (err) { + return callback(err); + } - async.filter(plugins, file.exists, next); - }, - ], callback); - }; + Object.assign(Plugins.staticDirs, results.staticDirs || {}); + add(Plugins.cssFiles, results.cssFiles); + add(Plugins.lessFiles, results.lessFiles); + add(Plugins.clientScripts, results.clientScripts); + add(Plugins.acpScripts, results.acpScripts); + Object.assign(meta.js.scripts.modules, results.modules || {}); + if (results.soundpack) { + Plugins.soundpacks.push(results.soundpack); + } + + callback(); + }); + } Plugins.prepareForBuild = function (callback) { Plugins.cssFiles.length = 0; @@ -42,29 +75,18 @@ module.exports = function (Plugins) { Plugins.soundpacks.length = 0; async.waterfall([ - async.apply(Plugins.getPluginPaths), - function (paths, next) { - async.map(paths, function (path, next) { - Plugins.loadPluginInfo(path, next); - }, next); - }, + Plugins.data.getActive, function (plugins, next) { async.each(plugins, function (pluginData, next) { - async.parallel([ - async.apply(mapFiles, pluginData, 'css', 'cssFiles'), - async.apply(mapFiles, pluginData, 'less', 'lessFiles'), - async.apply(mapClientSideScripts, pluginData), - async.apply(mapClientModules, pluginData), - async.apply(mapStaticDirectories, pluginData, pluginData.path), - async.apply(mapSoundpack, pluginData), - ], next); + // TODO: only load the data that's needed for the build + registerPluginAssets(pluginData, true, next); }, next); }, ], callback); }; Plugins.loadPlugin = function (pluginPath, callback) { - Plugins.loadPluginInfo(pluginPath, function (err, pluginData) { + Plugins.data.loadPluginInfo(pluginPath, function (err, pluginData) { if (err) { if (err.message === '[[error:parse-error]]') { return callback(); @@ -76,25 +98,13 @@ module.exports = function (Plugins) { async.parallel([ function (next) { - registerHooks(pluginData, pluginPath, next); - }, - function (next) { - mapStaticDirectories(pluginData, pluginPath, next); - }, - function (next) { - mapFiles(pluginData, 'css', 'cssFiles', next); - }, - function (next) { - mapFiles(pluginData, 'less', 'lessFiles', next); - }, - function (next) { - mapClientSideScripts(pluginData, next); + registerHooks(pluginData, next); }, function (next) { - mapClientModules(pluginData, next); - }, - function (next) { - mapSoundpack(pluginData, next); + // TODO: change this from `true` to `['soundpack']` + // this will skip several build-only plugin loading methods + // and only load soundpacks, which will speed up startup + registerPluginAssets(pluginData, true, next); }, ], function (err) { if (err) { @@ -124,12 +134,12 @@ module.exports = function (Plugins) { } } - function registerHooks(pluginData, pluginPath, callback) { + function registerHooks(pluginData, callback) { if (!pluginData.library) { return callback(); } - var libraryPath = path.join(pluginPath, pluginData.library); + var libraryPath = path.join(pluginData.path, pluginData.library); try { if (!Plugins.libraries[pluginData.id]) { @@ -149,199 +159,4 @@ module.exports = function (Plugins) { callback(); } } - - function mapStaticDirectories(pluginData, pluginPath, callback) { - var validMappedPath = /^[\w\-_]+$/; - - function mapStaticDirs(mappedPath, callback) { - if (Plugins.staticDirs[mappedPath]) { - winston.warn('[plugins/' + pluginData.id + '] Mapped path (' + mappedPath + ') already specified!'); - callback(); - } else if (!validMappedPath.test(mappedPath)) { - winston.warn('[plugins/' + pluginData.id + '] Invalid mapped path specified: ' + mappedPath + '. Path must adhere to: ' + validMappedPath.toString()); - callback(); - } else { - var realPath = pluginData.staticDirs[mappedPath]; - var staticDir = path.join(pluginPath, realPath); - - file.exists(staticDir, function (err, exists) { - if (exists) { - Plugins.staticDirs[pluginData.id + '/' + mappedPath] = staticDir; - } else { - winston.warn('[plugins/' + pluginData.id + '] Mapped path \'' + mappedPath + ' => ' + staticDir + '\' not found.'); - } - callback(err); - }); - } - } - - pluginData.staticDirs = pluginData.staticDirs || {}; - - var dirs = Object.keys(pluginData.staticDirs); - async.each(dirs, mapStaticDirs, callback); - } - - function mapFiles(pluginData, type, globalArray, callback) { - if (Array.isArray(pluginData[type])) { - if (global.env === 'development') { - winston.verbose('[plugins] Found ' + pluginData[type].length + ' ' + type + ' file(s) for plugin ' + pluginData.id); - } - - Plugins[globalArray] = Plugins[globalArray].concat(pluginData[type].map(function (file) { - return path.join(pluginData.id, file); - })); - } - callback(); - } - - function mapClientSideScripts(pluginData, callback) { - function mapScripts(scripts, param) { - if (Array.isArray(scripts) && scripts.length) { - if (global.env === 'development') { - winston.verbose('[plugins] Found ' + scripts.length + ' js file(s) for plugin ' + pluginData.id); - } - - Plugins[param] = Plugins[param].concat(scripts.map(function (file) { - return resolveModulePath(path.join(__dirname, '../../node_modules/', pluginData.id, file), file); - })).filter(Boolean); - } - } - mapScripts(pluginData.scripts, 'clientScripts'); - mapScripts(pluginData.acpScripts, 'acpScripts'); - - callback(); - } - - function mapClientModules(pluginData, callback) { - if (!pluginData.hasOwnProperty('modules')) { - return callback(); - } - - var modules = {}; - - if (Array.isArray(pluginData.modules)) { - if (global.env === 'development') { - winston.verbose('[plugins] Found ' + pluginData.modules.length + ' AMD-style module(s) for plugin ' + pluginData.id); - } - - var strip = pluginData.hasOwnProperty('modulesStrip') ? parseInt(pluginData.modulesStrip, 10) : 0; - - pluginData.modules.forEach(function (file) { - if (strip) { - modules[file.replace(new RegExp('.?(/[^/]+){' + strip + '}/'), '')] = path.join('./node_modules/', pluginData.id, file); - } else { - modules[path.basename(file)] = path.join('./node_modules/', pluginData.id, file); - } - }); - - meta.js.scripts.modules = _.extend(meta.js.scripts.modules, modules); - } else { - var keys = Object.keys(pluginData.modules); - - if (global.env === 'development') { - winston.verbose('[plugins] Found ' + keys.length + ' AMD-style module(s) for plugin ' + pluginData.id); - } - - for (var name in pluginData.modules) { - if (pluginData.modules.hasOwnProperty(name)) { - modules[name] = path.join('./node_modules/', pluginData.id, pluginData.modules[name]); - } - } - - meta.js.scripts.modules = _.extend(meta.js.scripts.modules, modules); - } - - callback(); - } - - function mapSoundpack(pluginData, callback) { - var soundpack = pluginData.soundpack; - if (!soundpack || !soundpack.dir || !soundpack.sounds) { - return callback(); - } - soundpack.name = soundpack.name || pluginData.name; - soundpack.id = pluginData.id; - soundpack.dir = path.join(pluginData.path, soundpack.dir); - async.each(Object.keys(soundpack.sounds), function (key, next) { - file.exists(path.join(soundpack.dir, soundpack.sounds[key]), function (err, exists) { - if (!exists) { - delete soundpack.sounds[key]; - } - - next(err); - }); - }, function (err) { - if (err) { - return callback(err); - } - - if (Object.keys(soundpack.sounds).length) { - Plugins.soundpacks.push(soundpack); - } - - callback(); - }); - } - - function resolveModulePath(fullPath, relPath) { - /** - * With npm@3, dependencies can become flattened, and appear at the root level. - * This method resolves these differences if it can. - */ - var matches = fullPath.match(/node_modules/g); - var atRootLevel = !matches || matches.length === 1; - - try { - fs.statSync(fullPath); - winston.verbose('[plugins/load] File found: ' + fullPath); - return fullPath; - } catch (e) { - // File not visible to the calling process, ascend to root level if possible and try again - if (!atRootLevel && relPath) { - winston.verbose('[plugins/load] File not found: ' + fullPath + ' (Ascending)'); - return resolveModulePath(path.join(__dirname, '../..', relPath)); - } - // Already at root level, file was simply not found - winston.warn('[plugins/load] File not found: ' + fullPath + ' (Ignoring)'); - return null; - } - } - - Plugins.loadPluginInfo = function (pluginPath, callback) { - async.parallel({ - package: function (next) { - fs.readFile(path.join(pluginPath, 'package.json'), next); - }, - plugin: function (next) { - fs.readFile(path.join(pluginPath, 'plugin.json'), next); - }, - }, function (err, results) { - if (err) { - return callback(err); - } - var pluginData; - var packageData; - try { - pluginData = JSON.parse(results.plugin); - packageData = JSON.parse(results.package); - - pluginData.id = packageData.name; - pluginData.name = packageData.name; - pluginData.description = packageData.description; - pluginData.version = packageData.version; - pluginData.repository = packageData.repository; - pluginData.nbbpm = packageData.nbbpm; - pluginData.path = pluginPath; - } catch (err) { - var pluginDir = pluginPath.split(path.sep); - pluginDir = pluginDir[pluginDir.length - 1]; - - winston.error('[plugins/' + pluginDir + '] Error in plugin.json or package.json! ' + err.message); - - return callback(new Error('[[error:parse-error]]')); - } - - callback(null, pluginData); - }); - }; };