Support npm@5 and yarn (#6010)

* Support npm@5 and yarn

Use package.default.json
Partial #6008

- Overwrite package.json with package.default.json values
- `dependencies` field is merged with package.default.json version taking precidence
- `./nodebb upgrade` automatically does those things and runs `git pull`
- use `./nodebb upgrade --dev` to avoid the `git pull`

* added logic to preserve extraneous plugins installed in node_modules/

* Don't automatically git pull

* Simplify package-install, run it on upgrade just in case

.gitignore

@@ -65,3 +65,4 @@ build
 test/files/normalise.jpg.png
 test/files/normalise-resized.jpg
 package-lock.json
+package.json

.travis.yml

@@ -8,6 +8,8 @@ before_install:
   - "sudo service mongod start"
 before_script:
   - sleep 15 # wait for mongodb to be ready
+  - cp package.default.json package.json
+  - npm install
   - sh -c "if [ '$DB' = 'mongodb' ]; then node app --setup=\"{\\\"url\\\":\\\"http://127.0.0.1:4567\\\",\\\"secret\\\":\\\"abcdef\\\",\\\"database\\\":\\\"mongo\\\",\\\"mongo:host\\\":\\\"127.0.0.1\\\",\\\"mongo:port\\\":27017,\\\"mongo:username\\\":\\\"\\\",\\\"mongo:password\\\":\\\"\\\",\\\"mongo:database\\\":0,\\\"redis:host\\\":\\\"127.0.0.1\\\",\\\"redis:port\\\":6379,\\\"redis:password\\\":\\\"\\\",\\\"redis:database\\\":0,\\\"admin:username\\\":\\\"admin\\\",\\\"admin:email\\\":\\\"test@example.org\\\",\\\"admin:password\\\":\\\"abcdef\\\",\\\"admin:password:confirm\\\":\\\"abcdef\\\"}\" --ci=\"{\\\"host\\\":\\\"127.0.0.1\\\",\\\"port\\\":27017,\\\"database\\\":0}\"; fi"
   - sh -c "if [ '$DB' = 'redis' ]; then node app --setup=\"{\\\"url\\\":\\\"http://127.0.0.1:4567\\\",\\\"secret\\\":\\\"abcdef\\\",\\\"database\\\":\\\"redis\\\",\\\"mongo:host\\\":\\\"127.0.0.1\\\",\\\"mongo:port\\\":27017,\\\"mongo:username\\\":\\\"\\\",\\\"mongo:password\\\":\\\"\\\",\\\"mongo:database\\\":0,\\\"redis:host\\\":\\\"127.0.0.1\\\",\\\"redis:port\\\":6379,\\\"redis:password\\\":\\\"\\\",\\\"redis:database\\\":0,\\\"admin:username\\\":\\\"admin\\\",\\\"admin:email\\\":\\\"test@example.org\\\",\\\"admin:password\\\":\\\"abcdef\\\",\\\"admin:password:confirm\\\":\\\"abcdef\\\"}\" --ci=\"{\\\"host\\\":\\\"127.0.0.1\\\",\\\"port\\\":6379,\\\"database\\\":0}\"; fi"
 after_success:

nodebb

@@ -6,18 +6,20 @@ var fs = require('fs');
 var path = require('path');
 var cproc = require('child_process');
 
+var packageInstall = require('./src/meta/package-install');
+
 // check to make sure dependencies are installed
 try {
+	fs.readFileSync(path.join(__dirname, './package.json'));
 	fs.readFileSync(path.join(__dirname, 'node_modules/async/package.json'));
 } catch (e) {
 	if (e.code === 'ENOENT') {
 		process.stdout.write('Dependencies not yet installed.\n');
 		process.stdout.write('Installing them now...\n\n');
 
-		cproc.execSync('npm i --production', {
-			cwd: __dirname,
-			stdio: [0, 1, 2],
-		});
+		packageInstall.updatePackageFile();
+		packageInstall.preserveExtraneousPlugins();
+		packageInstall.npmInstallProduction();
 	} else {
 		throw e;
 	}
@@ -458,9 +460,15 @@ var commands = {
 			}
 
 			async.series([
+				function (next) {
+					packageInstall.updatePackageFile();
+					packageInstall.preserveExtraneousPlugins();
+					next();
+				},
 				function (next) {
 					process.stdout.write('1. '.bold + 'Bringing base dependencies up to date... '.yellow);
-					cproc.exec('npm i --production', { cwd: __dirname, stdio: 'ignore' }, next);
+					packageInstall.npmInstallProduction();
+					next();
 				},
 				function (next) {
 					process.stdout.write('OK\n'.green);

src/meta/package-install.js

@@ -0,0 +1,72 @@
+'use strict';
+
+var path = require('path');
+var fs = require('fs');
+var cproc = require('child_process');
+
+var packageFilePath = path.join(__dirname, '../../package.json');
+var packageDefaultFilePath = path.join(__dirname, '../../package.default.json');
+var modulesPath = path.join(__dirname, '../../node_modules');
+
+function updatePackageFile() {
+	var oldPackageContents = {};
+
+	try {
+		oldPackageContents = JSON.parse(fs.readFileSync(packageFilePath, 'utf8'));
+	} catch (e) {
+		if (e.code !== 'ENOENT') {
+			throw e;
+		}
+	}
+
+	var defaultPackageContents = JSON.parse(fs.readFileSync(packageDefaultFilePath, 'utf8'));
+	var packageContents = Object.assign({}, oldPackageContents, defaultPackageContents, {
+		dependencies: Object.assign({}, oldPackageContents.dependencies, defaultPackageContents.dependencies),
+	});
+
+	fs.writeFileSync(packageFilePath, JSON.stringify(packageContents, null, 2));
+}
+
+exports.updatePackageFile = updatePackageFile;
+
+function npmInstallProduction() {
+	cproc.execSync('npm i --production', {
+		cwd: path.join(__dirname, '../../'),
+		stdio: [0, 1, 2],
+	});
+}
+
+exports.npmInstallProduction = npmInstallProduction;
+
+function preserveExtraneousPlugins() {
+	// Skip if `node_modules/` is not found or inaccessible
+	try {
+		fs.accessSync(modulesPath, fs.constants.R_OK);
+	} catch (e) {
+		return;
+	}
+
+	var isPackage = /^nodebb-(plugin|theme|widget|reward)-\w+/;
+	var packages = fs.readdirSync(modulesPath).filter(function (pkgName) {
+		return isPackage.test(pkgName);
+	});
+	var packageContents = JSON.parse(fs.readFileSync(packageFilePath, 'utf8'));
+
+	var extraneous = packages
+		// only extraneous plugins (ones not in package.json)
+		.filter(function (pkgName) {
+			return !packageContents.dependencies.hasOwnProperty(pkgName);
+		})
+		// reduce to a map of package names to package versions
+		.reduce(function (map, pkgName) {
+			var pkgConfig = JSON.parse(fs.readFileSync(path.join(modulesPath, pkgName, 'package.json')));
+			map[pkgName] = pkgConfig.version;
+			return map;
+		}, {});
+
+	// Add those packages to package.json
+	Object.assign(packageContents.dependencies, extraneous);
+	fs.writeFileSync(packageFilePath, JSON.stringify(packageContents, null, 2));
+}
+
+exports.preserveExtraneousPlugins = preserveExtraneousPlugins;

src/plugins/install.js

@@ -6,6 +6,7 @@ var path = require('path');
 var fs = require('fs');
 var nconf = require('nconf');
 var os = require('os');
+var cproc = require('child_process');
 
 var db = require('../database');
 var meta = require('../meta');
@@ -107,7 +108,7 @@ module.exports = function (Plugins) {
 	}
 
 	function runNpmCommand(command, pkgName, version, callback) {
-		require('child_process').execFile((process.platform === 'win32') ? 'npm.cmd' : 'npm', [command, pkgName + (command === 'install' ? '@' + version : ''), '--no-save'], function (err, stdout) {
+		cproc.execFile((process.platform === 'win32') ? 'npm.cmd' : 'npm', [command, pkgName + (command === 'install' ? '@' + version : ''), '--save'], function (err, stdout) {
 			if (err) {
 				return callback(err);
 			}

test/plugins.js

@@ -5,6 +5,7 @@ var	assert = require('assert');
 var path = require('path');
 var nconf = require('nconf');
 var request = require('request');
+var fs = require('fs');
 
 var db = require('./mocks/databasemock');
 var plugins = require('../src/plugins');
@@ -128,6 +129,9 @@ describe('Plugins', function () {
 				assert.equal(pluginData.active, false);
 				assert.equal(pluginData.installed, true);
 
+				var packageFile = JSON.parse(fs.readFileSync(path.join(__dirname, '../package.json'), 'utf8'));
+				assert(packageFile.dependencies[pluginName]);
+
 				done();
 			});
 		});
@@ -160,6 +164,10 @@ describe('Plugins', function () {
 				assert.ifError(err);
 				assert.equal(pluginData.installed, false);
 				assert.equal(pluginData.active, false);
+
+				var packageFile = JSON.parse(fs.readFileSync(path.join(__dirname, '../package.json'), 'utf8'));
+				assert(!packageFile.dependencies[pluginName]);
+
 				done();
 			});
 		});