From 5b8c9503c314aac3eece30c6abca98c634895a98 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 22 Feb 2017 13:20:47 -0500 Subject: [PATCH 01/16] refactor upgrade scripts to use individual files in src/upgrades/ as source of schema changes --- app.js | 16 +- nodebb | 12 + src/upgrade.js | 463 ++---------------- src/upgrades/20160922-category-recent-tids.js | 33 ++ .../20161008-favourites-to-bookmarks.js | 52 ++ .../20161014-sorted-sets-for-post-replies.js | 34 ++ .../20161122-global-and-user-language-keys.js | 63 +++ .../20161125-sorted-set-for-pinned-topics.js | 37 ++ src/upgrades/20161207-flags-refactor.js | 89 ++++ 9 files changed, 377 insertions(+), 422 deletions(-) create mode 100644 src/upgrades/20160922-category-recent-tids.js create mode 100644 src/upgrades/20161008-favourites-to-bookmarks.js create mode 100644 src/upgrades/20161014-sorted-sets-for-post-replies.js create mode 100644 src/upgrades/20161122-global-and-user-language-keys.js create mode 100644 src/upgrades/20161125-sorted-set-for-pinned-topics.js create mode 100644 src/upgrades/20161207-flags-refactor.js diff --git a/app.js b/app.js index 57c57d61b6..271e16fee5 100644 --- a/app.js +++ b/app.js @@ -189,13 +189,17 @@ function upgrade() { var meta = require('./src/meta'); var upgrade = require('./src/upgrade'); var build = require('./src/meta/build'); + var tasks = [db.init, meta.configs.init, upgrade.run, build.buildAll]; - async.series([ - async.apply(db.init), - async.apply(meta.configs.init), - async.apply(upgrade.upgrade), - async.apply(build.buildAll) - ], function (err) { + if (nconf.get('upgrade') !== true) { + // Likely an upgrade script name passed in + tasks[2] = async.apply(upgrade.runSingle, nconf.get('upgrade')); + + // Skip build + tasks.pop(); + } + + async.series(tasks, function (err) { if (err) { winston.error(err.stack); process.exit(1); diff --git a/nodebb b/nodebb index ee87b24288..8ee1628855 100755 --- a/nodebb +++ b/nodebb @@ -406,6 +406,18 @@ var commands = { description: 'Run NodeBB upgrade scripts, ensure packages are up-to-date', usage: 'Usage: ' + './nodebb upgrade'.yellow, handler: function () { + if (process.argv[3]) { + process.stdout.write('\nUpdating NodeBB data store schema...\n'.yellow); + var arr = ['--upgrade'].concat(process.argv.slice(3)); + var upgradeProc = fork(arr); + + return upgradeProc.on('close', function (err) { + if (err) { + process.stdout.write('\nError'.red + ': ' + err.message + '\n'); + } + }); + } + async.series([ function (next) { process.stdout.write('1. '.bold + 'Bringing base dependencies up to date... '.yellow); diff --git a/src/upgrade.js b/src/upgrade.js index 12939ad99d..1aee44d060 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -1,443 +1,74 @@ -"use strict"; +/* jslint node: true */ +'use strict'; -/* globals console, require */ - -var db = require('./database'); var async = require('async'); -var winston = require('winston'); +var path = require('path'); -var Upgrade = {}; +var utils = require('../public/src/utils'); -var minSchemaDate = Date.UTC(2016, 8, 7); // This value gets updated every new MAJOR version -var schemaDate; -var thisSchemaDate; +var Upgrade = {}; -// IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema -var latestSchema = Date.UTC(2016, 11, 7); +Upgrade.run = function (callback) { + process.stdout.write('\nParsing upgrade scripts... '); -Upgrade.check = function (callback) { - db.get('schemaDate', function (err, value) { + utils.walk(path.join(__dirname, './upgrades'), function (err, files) { if (err) { return callback(err); } - if (!value) { - db.set('schemaDate', latestSchema, function (err) { - if (err) { - return callback(err); - } - callback(null); - }); - return; - } - - var schema_ok = parseInt(value, 10) >= latestSchema; - callback(!schema_ok ? new Error('schema-out-of-date') : null); + Upgrade.process(files, callback); }); }; -Upgrade.update = function (schemaDate, callback) { - db.set('schemaDate', schemaDate, callback); -}; - -Upgrade.upgrade = function (callback) { - var updatesMade = false; - - winston.info('Beginning database schema update'); - - async.series([ - function (next) { - // Prepare for upgrade & check to make sure the upgrade is possible - db.get('schemaDate', function (err, value) { - if (err) { - return next(err); - } - - if(!value) { - db.set('schemaDate', latestSchema, function () { - next(); - }); - schemaDate = latestSchema; - } else { - schemaDate = parseInt(value, 10); - } - - if (schemaDate >= minSchemaDate) { - next(); - } else { - next(new Error('upgrade-not-possible')); - } - }); - }, - function (next) { - thisSchemaDate = Date.UTC(2016, 8, 22); - - if (schemaDate < thisSchemaDate) { - updatesMade = true; - winston.info('[2016/09/22] Setting category recent tids'); - - - db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) { - if (err) { - return next(err); - } - - async.eachSeries(cids, function (cid, next) { - db.getSortedSetRevRange('cid:' + cid + ':pids', 0, 0, function (err, pid) { - if (err || !pid) { - return next(err); - } - db.getObjectFields('post:' + pid, ['tid', 'timestamp'], function (err, postData) { - if (err || !postData || !postData.tid) { - return next(err); - } - db.sortedSetAdd('cid:' + cid + ':recent_tids', postData.timestamp, postData.tid, next); - }); - }); - }, function (err) { - if (err) { - return next(err); - } - - winston.info('[2016/09/22] Setting category recent tids - done'); - Upgrade.update(thisSchemaDate, next); - }); - }); - } else { - winston.info('[2016/09/22] Setting category recent tids - skipped!'); - next(); - } - }, - function (next) { - function upgradePosts(next) { - var batch = require('./batch'); - - batch.processSortedSet('posts:pid', function (ids, next) { - async.each(ids, function (id, next) { - console.log('processing pid ' + id); - async.waterfall([ - function (next) { - db.rename('pid:' + id + ':users_favourited', 'pid:' + id + ':users_bookmarked', next); - }, - function (next) { - db.getObjectField('post:' + id, 'reputation', next); - }, - function (reputation, next) { - if (parseInt(reputation, 10)) { - db.setObjectField('post:' + id, 'bookmarks', reputation, next); - } else { - next(); - } - }, - function (next) { - db.deleteObjectField('post:' + id, 'reputation', next); - } - ], next); - }, next); - }, {}, next); - } - - function upgradeUsers(next) { - var batch = require('./batch'); - - batch.processSortedSet('users:joindate', function (ids, next) { - async.each(ids, function (id, next) { - console.log('processing uid ' + id); - db.rename('uid:' + id + ':favourites', 'uid:' + id + ':bookmarks', next); - }, next); - }, {}, next); - } - - thisSchemaDate = Date.UTC(2016, 9, 8); - - if (schemaDate < thisSchemaDate) { - updatesMade = true; - winston.info('[2016/10/8] favourite -> bookmark refactor'); - async.series([upgradePosts, upgradeUsers], function (err) { - if (err) { - return next(err); - } - winston.info('[2016/08/05] favourite- bookmark refactor done!'); - Upgrade.update(thisSchemaDate, next); - }); - } else { - winston.info('[2016/10/8] favourite -> bookmark refactor - skipped!'); - next(); - } - }, - function (next) { - thisSchemaDate = Date.UTC(2016, 9, 14); - - if (schemaDate < thisSchemaDate) { - updatesMade = true; - winston.info('[2016/10/14] Creating sorted sets for post replies'); - - var posts = require('./posts'); - var batch = require('./batch'); - batch.processSortedSet('posts:pid', function (ids, next) { - posts.getPostsFields(ids, ['pid', 'toPid', 'timestamp'], function (err, data) { - if (err) { - return next(err); - } - - async.eachSeries(data, function (postData, next) { - if (!parseInt(postData.toPid, 10)) { - return next(null); - } - console.log('processing pid: ' + postData.pid + ' toPid: ' + postData.toPid); - async.parallel([ - async.apply(db.sortedSetAdd, 'pid:' + postData.toPid + ':replies', postData.timestamp, postData.pid), - async.apply(db.incrObjectField, 'post:' + postData.toPid, 'replies') - ], next); - }, next); - }); - }, function (err) { - if (err) { - return next(err); - } - - winston.info('[2016/10/14] Creating sorted sets for post replies - done'); - Upgrade.update(thisSchemaDate, next); - }); - } else { - winston.info('[2016/10/14] Creating sorted sets for post replies - skipped!'); - next(); - } - }, - function (next) { - thisSchemaDate = Date.UTC(2016, 10, 22); - - if (schemaDate < thisSchemaDate) { - updatesMade = true; - winston.info('[2016/11/22] Update global and user language keys'); - - var user = require('./user'); - var meta = require('./meta'); - var batch = require('./batch'); - var newLanguage; - var i = 0; - var j = 0; - async.parallel([ - function (next) { - meta.configs.get('defaultLang', function (err, defaultLang) { - if (err) { - return next(err); - } - - if (!defaultLang) { - return setImmediate(next); - } - - newLanguage = defaultLang.replace('_', '-').replace('@', '-x-'); - if (newLanguage !== defaultLang) { - meta.configs.set('defaultLang', newLanguage, next); - } else { - setImmediate(next); - } - }); - }, - function (next) { - batch.processSortedSet('users:joindate', function (ids, next) { - async.each(ids, function (uid, next) { - async.waterfall([ - async.apply(db.getObjectField, 'user:' + uid + ':settings', 'userLang'), - function (language, next) { - ++i; - if (!language) { - return setImmediate(next); - } - - newLanguage = language.replace('_', '-').replace('@', '-x-'); - if (newLanguage !== language) { - ++j; - user.setSetting(uid, 'userLang', newLanguage, next); - } else { - setImmediate(next); - } - } - ], next); - }, next); - }, next); - } - ], function (err) { - if (err) { - return next(err); - } - - winston.info('[2016/11/22] Update global and user language keys - done (' + i + ' processed, ' + j + ' updated)'); - Upgrade.update(thisSchemaDate, next); - }); - } else { - winston.info('[2016/11/22] Update global and user language keys - skipped!'); - next(); - } - }, - function (next) { - thisSchemaDate = Date.UTC(2016, 10, 25); - - if (schemaDate < thisSchemaDate) { - updatesMade = true; - winston.info('[2016/11/25] Creating sorted sets for pinned topics'); - - var topics = require('./topics'); - var batch = require('./batch'); - batch.processSortedSet('topics:tid', function (ids, next) { - topics.getTopicsFields(ids, ['tid', 'cid', 'pinned', 'lastposttime'], function (err, data) { - if (err) { - return next(err); - } - - data = data.filter(function (topicData) { - return parseInt(topicData.pinned, 10) === 1; - }); - - async.eachSeries(data, function (topicData, next) { - console.log('processing tid: ' + topicData.tid); - - async.parallel([ - async.apply(db.sortedSetAdd, 'cid:' + topicData.cid + ':tids:pinned', Date.now(), topicData.tid), - async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids', topicData.tid), - async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids:posts', topicData.tid) - ], next); - }, next); - }); - }, function (err) { - if (err) { - return next(err); - } - - winston.info('[2016/11/25] Creating sorted sets for pinned topics - done'); - Upgrade.update(thisSchemaDate, next); - }); - } else { - winston.info('[2016/11/25] Creating sorted sets for pinned topics - skipped!'); - next(); - } - }, - function (next) { - thisSchemaDate = Date.UTC(2016, 11, 7); +Upgrade.runSingle = function (query, callback) { + process.stdout.write('\nParsing upgrade scripts... '); - if (schemaDate < thisSchemaDate) { - updatesMade = true; - winston.info('[2016/12/07] Migrating flags to new schema (#5232)'); - - var batch = require('./batch'); - var posts = require('./posts'); - var flags = require('./flags'); - var migrated = 0; - - batch.processSortedSet('posts:pid', function (ids, next) { - posts.getPostsByPids(ids, 1, function (err, posts) { - if (err) { - return next(err); - } + async.waterfall([ + async.apply(utils.walk, path.join(__dirname, './upgrades')), + function (files, next) { + next(null, files.filter(function (file) { + return file.search(new RegExp(query)) !== -1; + })); + } + ], function (err, files) { + if (err) { + return callback(err); + } - posts = posts.filter(function (post) { - return post.hasOwnProperty('flags'); - }); - - async.each(posts, function (post, next) { - async.parallel({ - uids: async.apply(db.getSortedSetRangeWithScores, 'pid:' + post.pid + ':flag:uids', 0, -1), - reasons: async.apply(db.getSortedSetRange, 'pid:' + post.pid + ':flag:uid:reason', 0, -1) - }, function (err, data) { - if (err) { - return next(err); - } + Upgrade.process(files, callback); + }); +}; - // Adding in another check here in case a post was improperly dismissed (flag count > 1 but no flags in db) - if (!data.uids.length || !data.reasons.length) { - return setImmediate(next); - } +Upgrade.process = function (files, callback) { + process.stdout.write('OK'.green + String(' ' + files.length).cyan + ' script(s) found\n'.cyan); - // Just take the first entry - var datetime = data.uids[0].score; - var reason = data.reasons[0].split(':')[1]; - var flagObj; + // Do I need to sort the files here? we'll see. + // sort(); - async.waterfall([ - async.apply(flags.create, 'post', post.pid, data.uids[0].value, reason, datetime), - function (_flagObj, next) { - flagObj = _flagObj; - if (post['flag:state'] || post['flag:assignee']) { - flags.update(flagObj.flagId, 1, { - state: post['flag:state'], - assignee: post['flag:assignee'], - datetime: datetime - }, next); - } else { - setImmediate(next); - } - }, - function (next) { - if (post.hasOwnProperty('flag:notes') && post['flag:notes'].length) { - try { - var history = JSON.parse(post['flag:history']); - history = history.filter(function (event) { - return event.type === 'notes'; - })[0]; + async.eachSeries(files, function (file, next) { + var scriptExport = require(file); + var date = new Date(scriptExport.timestamp); - flags.appendNote(flagObj.flagId, history.uid, post['flag:notes'], history.timestamp, next); - } catch (e) { - next(e); - } - } else { - setImmediate(next); - } - } - ], function (err) { - if (err && err.message === '[[error:already-flagged]]') { - // Already flagged, no need to parse, but not an error - next(); - } else { - next(err); - } - }); - }); - }, next); - }); - }, function (err) { - if (err) { - return next(err); - } + process.stdout.write(' → '.white + String('[' + [date.getFullYear(), date.getMonth() + 1, date.getDate() + 1].join('/') + '] ').gray + String(scriptExport.name).reset + '... '); - winston.info('[2016/12/07] Migrating flags to new schema (#5232) - done'); - Upgrade.update(thisSchemaDate, next); - }); - } else { - winston.info('[2016/12/07] Migrating flags to new schema (#5232) - skipped!'); - next(); - } - } - // Add new schema updates here - // IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema IN LINE 24!!! - ], function (err) { - if (!err) { - if(updatesMade) { - winston.info('[upgrade] Schema update complete!'); - } else { - winston.info('[upgrade] Schema already up to date!'); + // Do the upgrade... + scriptExport.method(function (err) { + if (err) { + process.stdout.write('error\n'.red); + return next(err); } - } else { - switch(err.message) { - case 'upgrade-not-possible': - winston.error('[upgrade] NodeBB upgrade could not complete, as your database schema is too far out of date.'); - winston.error('[upgrade] Please ensure that you did not skip any minor version upgrades.'); - winston.error('[upgrade] (e.g. v0.1.x directly to v0.3.x)'); - break; - default: - winston.error('[upgrade] Errors were encountered while updating the NodeBB schema: ' + err.message); - break; - } + process.stdout.write('OK\n'.green); + next(); + }); + }, function (err) { + if (err) { + return callback(err); } - if (typeof callback === 'function') { - callback(err); - } else { - process.exit(); - } + process.stdout.write('Upgrade complete!\n\n'.green); + callback(); }); }; -module.exports = Upgrade; +module.exports = Upgrade; \ No newline at end of file diff --git a/src/upgrades/20160922-category-recent-tids.js b/src/upgrades/20160922-category-recent-tids.js new file mode 100644 index 0000000000..ccb6f5c818 --- /dev/null +++ b/src/upgrades/20160922-category-recent-tids.js @@ -0,0 +1,33 @@ +/* jslint node: true */ +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Category recent tids', + timestamp: Date.UTC(2016, 8, 22), + method: function (callback) { + db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) { + if (err) { + return callback(err); + } + + async.eachSeries(cids, function (cid, next) { + db.getSortedSetRevRange('cid:' + cid + ':pids', 0, 0, function (err, pid) { + if (err || !pid) { + return next(err); + } + db.getObjectFields('post:' + pid, ['tid', 'timestamp'], function (err, postData) { + if (err || !postData || !postData.tid) { + return next(err); + } + db.sortedSetAdd('cid:' + cid + ':recent_tids', postData.timestamp, postData.tid, next); + }); + }); + }, callback); + }); + } +}; \ No newline at end of file diff --git a/src/upgrades/20161008-favourites-to-bookmarks.js b/src/upgrades/20161008-favourites-to-bookmarks.js new file mode 100644 index 0000000000..288d32c511 --- /dev/null +++ b/src/upgrades/20161008-favourites-to-bookmarks.js @@ -0,0 +1,52 @@ +/* jslint node: true */ +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Favourites to Bookmarks', + timestamp: Date.UTC(2016, 9, 8), + method: function (callback) { + function upgradePosts(next) { + var batch = require('../batch'); + + batch.processSortedSet('posts:pid', function (ids, next) { + async.each(ids, function (id, next) { + async.waterfall([ + function (next) { + db.rename('pid:' + id + ':users_favourited', 'pid:' + id + ':users_bookmarked', next); + }, + function (next) { + db.getObjectField('post:' + id, 'reputation', next); + }, + function (reputation, next) { + if (parseInt(reputation, 10)) { + db.setObjectField('post:' + id, 'bookmarks', reputation, next); + } else { + next(); + } + }, + function (next) { + db.deleteObjectField('post:' + id, 'reputation', next); + } + ], next); + }, next); + }, {}, next); + } + + function upgradeUsers(next) { + var batch = require('../batch'); + + batch.processSortedSet('users:joindate', function (ids, next) { + async.each(ids, function (id, next) { + db.rename('uid:' + id + ':favourites', 'uid:' + id + ':bookmarks', next); + }, next); + }, {}, next); + } + + async.series([upgradePosts, upgradeUsers], callback); + } +}; \ No newline at end of file diff --git a/src/upgrades/20161014-sorted-sets-for-post-replies.js b/src/upgrades/20161014-sorted-sets-for-post-replies.js new file mode 100644 index 0000000000..7442b614d4 --- /dev/null +++ b/src/upgrades/20161014-sorted-sets-for-post-replies.js @@ -0,0 +1,34 @@ +/* jslint node: true */ +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Sorted sets for post replies', + timestamp: Date.UTC(2016, 9, 14), + method: function (callback) { + var posts = require('../posts'); + var batch = require('../batch'); + batch.processSortedSet('posts:pid', function (ids, next) { + posts.getPostsFields(ids, ['pid', 'toPid', 'timestamp'], function (err, data) { + if (err) { + return next(err); + } + + async.eachSeries(data, function (postData, next) { + if (!parseInt(postData.toPid, 10)) { + return next(null); + } + winston.verbose('processing pid: ' + postData.pid + ' toPid: ' + postData.toPid); + async.parallel([ + async.apply(db.sortedSetAdd, 'pid:' + postData.toPid + ':replies', postData.timestamp, postData.pid), + async.apply(db.incrObjectField, 'post:' + postData.toPid, 'replies') + ], next); + }, next); + }); + }, callback); + } +}; \ No newline at end of file diff --git a/src/upgrades/20161122-global-and-user-language-keys.js b/src/upgrades/20161122-global-and-user-language-keys.js new file mode 100644 index 0000000000..d08b3fea74 --- /dev/null +++ b/src/upgrades/20161122-global-and-user-language-keys.js @@ -0,0 +1,63 @@ +/* jslint node: true */ +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Update global and user language keys', + timestamp: Date.UTC(2016, 10, 22), + method: function (callback) { + var user = require('../user'); + var meta = require('../meta'); + var batch = require('../batch'); + var newLanguage; + var i = 0; + var j = 0; + async.parallel([ + function (next) { + meta.configs.get('defaultLang', function (err, defaultLang) { + if (err) { + return next(err); + } + + if (!defaultLang) { + return setImmediate(next); + } + + newLanguage = defaultLang.replace('_', '-').replace('@', '-x-'); + if (newLanguage !== defaultLang) { + meta.configs.set('defaultLang', newLanguage, next); + } else { + setImmediate(next); + } + }); + }, + function (next) { + batch.processSortedSet('users:joindate', function (ids, next) { + async.each(ids, function (uid, next) { + async.waterfall([ + async.apply(db.getObjectField, 'user:' + uid + ':settings', 'userLang'), + function (language, next) { + ++i; + if (!language) { + return setImmediate(next); + } + + newLanguage = language.replace('_', '-').replace('@', '-x-'); + if (newLanguage !== language) { + ++j; + user.setSetting(uid, 'userLang', newLanguage, next); + } else { + setImmediate(next); + } + } + ], next); + }, next); + }, next); + } + ], callback); + } +}; \ No newline at end of file diff --git a/src/upgrades/20161125-sorted-set-for-pinned-topics.js b/src/upgrades/20161125-sorted-set-for-pinned-topics.js new file mode 100644 index 0000000000..d10a6ee902 --- /dev/null +++ b/src/upgrades/20161125-sorted-set-for-pinned-topics.js @@ -0,0 +1,37 @@ +/* jslint node: true */ +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Sorted set for pinned topics', + timestamp: Date.UTC(2016, 10, 25), + method: function (callback) { + var topics = require('../topics'); + var batch = require('../batch'); + batch.processSortedSet('topics:tid', function (ids, next) { + topics.getTopicsFields(ids, ['tid', 'cid', 'pinned', 'lastposttime'], function (err, data) { + if (err) { + return next(err); + } + + data = data.filter(function (topicData) { + return parseInt(topicData.pinned, 10) === 1; + }); + + async.eachSeries(data, function (topicData, next) { + winston.verbose('processing tid: ' + topicData.tid); + + async.parallel([ + async.apply(db.sortedSetAdd, 'cid:' + topicData.cid + ':tids:pinned', Date.now(), topicData.tid), + async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids', topicData.tid), + async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids:posts', topicData.tid) + ], next); + }, next); + }); + }, callback); + } +}; \ No newline at end of file diff --git a/src/upgrades/20161207-flags-refactor.js b/src/upgrades/20161207-flags-refactor.js new file mode 100644 index 0000000000..c61a3a4747 --- /dev/null +++ b/src/upgrades/20161207-flags-refactor.js @@ -0,0 +1,89 @@ +/* jslint node: true */ +'use strict'; + +var db = require('../database'); + +var async = require('async'); + +module.exports = { + name: 'Migrating flags to new schema', + timestamp: Date.UTC(2016, 11, 7), + method: function (callback) { + var batch = require('../batch'); + var posts = require('../posts'); + var flags = require('../flags'); + var migrated = 0; + + batch.processSortedSet('posts:pid', function (ids, next) { + posts.getPostsByPids(ids, 1, function (err, posts) { + if (err) { + return next(err); + } + + posts = posts.filter(function (post) { + return post.hasOwnProperty('flags'); + }); + + async.each(posts, function (post, next) { + async.parallel({ + uids: async.apply(db.getSortedSetRangeWithScores, 'pid:' + post.pid + ':flag:uids', 0, -1), + reasons: async.apply(db.getSortedSetRange, 'pid:' + post.pid + ':flag:uid:reason', 0, -1) + }, function (err, data) { + if (err) { + return next(err); + } + + // Adding in another check here in case a post was improperly dismissed (flag count > 1 but no flags in db) + if (!data.uids.length || !data.reasons.length) { + return setImmediate(next); + } + + // Just take the first entry + var datetime = data.uids[0].score; + var reason = data.reasons[0].split(':')[1]; + var flagObj; + + async.waterfall([ + async.apply(flags.create, 'post', post.pid, data.uids[0].value, reason, datetime), + function (_flagObj, next) { + flagObj = _flagObj; + if (post['flag:state'] || post['flag:assignee']) { + flags.update(flagObj.flagId, 1, { + state: post['flag:state'], + assignee: post['flag:assignee'], + datetime: datetime + }, next); + } else { + setImmediate(next); + } + }, + function (next) { + if (post.hasOwnProperty('flag:notes') && post['flag:notes'].length) { + try { + var history = JSON.parse(post['flag:history']); + history = history.filter(function (event) { + return event.type === 'notes'; + })[0]; + + flags.appendNote(flagObj.flagId, history.uid, post['flag:notes'], history.timestamp, next); + } catch (e) { + next(e); + } + } else { + setImmediate(next); + } + } + ], function (err) { + if (err && err.message === '[[error:already-flagged]]') { + // Already flagged, no need to parse, but not an error + next(); + } else { + next(err); + } + }); + }); + }, next); + }); + }, callback); + } +}; \ No newline at end of file From 9eed3abb1d9a47fa21741a0a7a6e09ad525e7374 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 22 Feb 2017 13:28:22 -0500 Subject: [PATCH 02/16] added template to upgrades folder --- src/upgrades/TEMPLATE | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/upgrades/TEMPLATE diff --git a/src/upgrades/TEMPLATE b/src/upgrades/TEMPLATE new file mode 100644 index 0000000000..5b79aa6148 --- /dev/null +++ b/src/upgrades/TEMPLATE @@ -0,0 +1,15 @@ +/* jslint node: true */ +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'User_friendly_upgrade_script_name', + timestamp: Date.UTC(2017, 0, 1), + method: function (callback) { + // Do stuff here... + } +}; \ No newline at end of file From 6ea0fc4e2f358cc7b9e5350b13a4f44023e5825f Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 22 Feb 2017 15:57:31 -0500 Subject: [PATCH 03/16] changed file schema to be name only, storing them in object sets referencing their applicable versions --- src/upgrade.js | 50 ++++++++++++++++--- ...recent-tids.js => category_recent_tids.js} | 0 ...ookmarks.js => favourites_to_bookmarks.js} | 0 ...07-flags-refactor.js => flags_refactor.js} | 0 ...ys.js => global_and_user_language_keys.js} | 0 ...ics.js => sorted_set_for_pinned_topics.js} | 0 ...ies.js => sorted_sets_for_post_replies.js} | 0 7 files changed, 44 insertions(+), 6 deletions(-) rename src/upgrades/{20160922-category-recent-tids.js => category_recent_tids.js} (100%) rename src/upgrades/{20161008-favourites-to-bookmarks.js => favourites_to_bookmarks.js} (100%) rename src/upgrades/{20161207-flags-refactor.js => flags_refactor.js} (100%) rename src/upgrades/{20161122-global-and-user-language-keys.js => global_and_user_language_keys.js} (100%) rename src/upgrades/{20161125-sorted-set-for-pinned-topics.js => sorted_set_for_pinned_topics.js} (100%) rename src/upgrades/{20161014-sorted-sets-for-post-replies.js => sorted_sets_for_post_replies.js} (100%) diff --git a/src/upgrade.js b/src/upgrade.js index 1aee44d060..be303b4876 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -4,19 +4,54 @@ var async = require('async'); var path = require('path'); +var db = require('./database'); var utils = require('../public/src/utils'); -var Upgrade = {}; +var Upgrade = { + available: [ + { + version: "1.2.0", + upgrades: ['category_recent_tids'] + }, + { + version: "1.3.0", + upgrades: ['favourites_to_bookmarks', 'sorted_sets_for_post_replies'] + }, + { + version: "1.4.0", + upgrades: ['global_and_user_language_keys', 'sorted_set_for_pinned_topics'] + }, + { + version: "1.5.0", + upgrades: ['flags_refactor'] + } + ] +}; Upgrade.run = function (callback) { process.stdout.write('\nParsing upgrade scripts... '); + var queue = []; + var skipped = 0; - utils.walk(path.join(__dirname, './upgrades'), function (err, files) { + // Retrieve list of upgrades that have already been run + db.getSortedSetRange('schemaLog', 0, -1, function (err, completed) { if (err) { return callback(err); } - Upgrade.process(files, callback); + queue = Upgrade.available.reduce(function (memo, cur) { + cur.upgrades.forEach(function (filename) { + if (completed.indexOf(filename) === -1) { + memo.push(path.join(__dirname, './upgrades', filename)); + } else { + ++skipped; + } + }); + + return memo; + }, queue); + + Upgrade.process(queue, skipped, callback); }); }; @@ -35,12 +70,12 @@ Upgrade.runSingle = function (query, callback) { return callback(err); } - Upgrade.process(files, callback); + Upgrade.process(files, 0, callback); }); }; -Upgrade.process = function (files, callback) { - process.stdout.write('OK'.green + String(' ' + files.length).cyan + ' script(s) found\n'.cyan); +Upgrade.process = function (files, skipCount, callback) { + process.stdout.write('OK'.green + ' | '.reset + String(files.length).cyan + ' script(s) found'.cyan + (skipCount > 0 ? ', '.cyan + String(skipCount).cyan + ' skipped'.cyan : '') + '\n'.reset); // Do I need to sort the files here? we'll see. // sort(); @@ -58,6 +93,9 @@ Upgrade.process = function (files, callback) { return next(err); } + // Record success in schemaLog + db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js')); + process.stdout.write('OK\n'.green); next(); }); diff --git a/src/upgrades/20160922-category-recent-tids.js b/src/upgrades/category_recent_tids.js similarity index 100% rename from src/upgrades/20160922-category-recent-tids.js rename to src/upgrades/category_recent_tids.js diff --git a/src/upgrades/20161008-favourites-to-bookmarks.js b/src/upgrades/favourites_to_bookmarks.js similarity index 100% rename from src/upgrades/20161008-favourites-to-bookmarks.js rename to src/upgrades/favourites_to_bookmarks.js diff --git a/src/upgrades/20161207-flags-refactor.js b/src/upgrades/flags_refactor.js similarity index 100% rename from src/upgrades/20161207-flags-refactor.js rename to src/upgrades/flags_refactor.js diff --git a/src/upgrades/20161122-global-and-user-language-keys.js b/src/upgrades/global_and_user_language_keys.js similarity index 100% rename from src/upgrades/20161122-global-and-user-language-keys.js rename to src/upgrades/global_and_user_language_keys.js diff --git a/src/upgrades/20161125-sorted-set-for-pinned-topics.js b/src/upgrades/sorted_set_for_pinned_topics.js similarity index 100% rename from src/upgrades/20161125-sorted-set-for-pinned-topics.js rename to src/upgrades/sorted_set_for_pinned_topics.js diff --git a/src/upgrades/20161014-sorted-sets-for-post-replies.js b/src/upgrades/sorted_sets_for_post_replies.js similarity index 100% rename from src/upgrades/20161014-sorted-sets-for-post-replies.js rename to src/upgrades/sorted_sets_for_post_replies.js From 35248c543b5175ee2559aa1631514bad29b6cc28 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sat, 25 Feb 2017 15:40:05 -0500 Subject: [PATCH 04/16] linting nodebb executable --- nodebb | 49 +++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/nodebb b/nodebb index 8ee1628855..f0fac9d0cf 100755 --- a/nodebb +++ b/nodebb @@ -29,7 +29,7 @@ if (args.dev) { function getRunningPid(callback) { fs.readFile(__dirname + '/pidfile', { - encoding: 'utf-8' + encoding: 'utf-8', }, function (err, pid) { if (err) { return callback(err); @@ -38,7 +38,7 @@ function getRunningPid(callback) { try { process.kill(parseInt(pid, 10), 0); callback(null, parseInt(pid, 10)); - } catch(e) { + } catch (e) { callback(e); } }); @@ -52,7 +52,7 @@ function getCurrentVersion(callback) { try { pkg = JSON.parse(pkg); return callback(null, pkg.version); - } catch(err) { + } catch (err) { return callback(err); } }); @@ -60,20 +60,21 @@ function getCurrentVersion(callback) { function fork(args) { return cproc.fork('app.js', args, { cwd: __dirname, - silent: false + silent: false, }); } function getInstalledPlugins(callback) { async.parallel({ files: async.apply(fs.readdir, path.join(__dirname, 'node_modules')), - deps: async.apply(fs.readFile, path.join(__dirname, 'package.json'), { encoding: 'utf-8' }) + deps: async.apply(fs.readFile, path.join(__dirname, 'package.json'), { encoding: 'utf-8' }), }, function (err, payload) { if (err) { return callback(err); } - var isNbbModule = /^nodebb-(?:plugin|theme|widget|rewards)-[\w\-]+$/, - moduleName, isGitRepo; + var isNbbModule = /^nodebb-(?:plugin|theme|widget|rewards)-[\w\-]+$/; + var moduleName; + var isGitRepo; payload.files = payload.files.filter(function (file) { return isNbbModule.test(file); @@ -98,7 +99,7 @@ function getInstalledPlugins(callback) { try { fs.accessSync(path.join(__dirname, 'node_modules/' + moduleName, '.git')); isGitRepo = true; - } catch(e) { + } catch (e) { isGitRepo = false; } @@ -144,7 +145,7 @@ function checkPlugins(standalone, callback) { async.waterfall([ async.apply(async.parallel, { plugins: async.apply(getInstalledPlugins), - version: async.apply(getCurrentVersion) + version: async.apply(getCurrentVersion), }), function (payload, next) { var toCheck = Object.keys(payload.plugins); @@ -157,7 +158,7 @@ function checkPlugins(standalone, callback) { request({ method: 'GET', url: 'https://packages.nodebb.org/api/v1/suggest?version=' + payload.version + '&package[]=' + toCheck.join('&package[]='), - json: true + json: true, }, function (err, res, body) { if (err) { process.stdout.write('error'.red + '\n'.reset); @@ -169,7 +170,8 @@ function checkPlugins(standalone, callback) { body = [body]; } - var current, suggested, + var current, + suggested, upgradable = body.map(function (suggestObj) { current = payload.plugins[suggestObj.package]; suggested = suggestObj.version; @@ -178,16 +180,15 @@ function checkPlugins(standalone, callback) { return { name: suggestObj.package, current: current, - suggested: suggested + suggested: suggested, }; - } else { - return null; } + return null; }).filter(Boolean); next(null, upgradable); }); - } + }, ], callback); } function upgradePlugins(callback) { @@ -223,7 +224,7 @@ function upgradePlugins(callback) { prompt.get({ name: 'upgrade', description: 'Proceed with upgrade (y|n)?'.reset, - type: 'string' + type: 'string', }, function (err, result) { if (err) { return callback(err); @@ -280,7 +281,7 @@ var commands = { // Spawn a new NodeBB process cproc.fork(__dirname + '/loader.js', { - env: process.env + env: process.env, }); }, }, @@ -320,7 +321,7 @@ var commands = { process.stdout.write('\n\n'.reset); cproc.spawn('tail', ['-F', './logs/output.log'], { cwd: __dirname, - stdio: 'inherit' + stdio: 'inherit', }); }, }, @@ -334,11 +335,11 @@ var commands = { // Spawn a new NodeBB process cproc.fork(__dirname + '/loader.js', { - env: process.env + env: process.env, }); cproc.spawn('tail', ['-F', './logs/output.log'], { cwd: __dirname, - stdio: 'inherit' + stdio: 'inherit', }); }, }, @@ -348,13 +349,13 @@ var commands = { handler: function () { process.env.NODE_ENV = 'development'; cproc.fork(__dirname + '/loader.js', ['--no-daemon', '--no-silent'], { - env: process.env + env: process.env, }); }, }, build: { description: 'Compile static assets (CSS, Javascript, etc)', - usage: 'Usage: ' + './nodebb build'.yellow + ' [js,clientCSS,acpCSS,tpl,lang]'.red + '\n' + + usage: 'Usage: ' + './nodebb build'.yellow + ' [js,clientCSS,acpCSS,tpl,lang]'.red + '\n' + ' e.g. ' + './nodebb build js,tpl'.yellow + '\tbuilds JS and templates\n' + ' ' + './nodebb build'.yellow + '\t\tbuilds all targets\n', handler: function () { @@ -434,7 +435,7 @@ var commands = { var upgradeProc = fork(arr); upgradeProc.on('close', next); - } + }, ], function (err) { if (err) { process.stdout.write('\nError'.red + ': ' + err.message + '\n'); @@ -442,7 +443,7 @@ var commands = { var message = 'NodeBB Upgrade Complete!'; // some consoles will return undefined/zero columns, so just use 2 spaces in upgrade script if we can't get our column count var columns = process.stdout.columns; - var spaces = columns ? new Array(Math.floor(columns / 2) - (message.length / 2) + 1).join(' ') : " "; + var spaces = columns ? new Array(Math.floor(columns / 2) - (message.length / 2) + 1).join(' ') : ' '; process.stdout.write('OK\n'.green); process.stdout.write('\n' + spaces + message.green.bold + '\n\n'.reset); From b385655dba522a2a1f3e1725395b00313cffe6a5 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sat, 25 Feb 2017 15:40:22 -0500 Subject: [PATCH 05/16] adding upgrade scripts from all of v1.x.x, #5467 --- src/upgrade.js | 16 ++- src/upgrades/assign_topic_read_privilege.js | 73 ++++++++++++ src/upgrades/chat_room_hashes.js | 41 +++++++ src/upgrades/chat_upgrade.js | 87 ++++++++++++++ .../dismiss_flags_from_deleted_topics.js | 43 +++++++ .../edit_delete_deletetopic_privileges.js | 109 ++++++++++++++++++ src/upgrades/global_moderators.js | 34 ++++++ src/upgrades/group_title_update.js | 34 ++++++ src/upgrades/remove_negative_best_posts.js | 22 ++++ src/upgrades/separate_upvote_downvote.js | 50 ++++++++ src/upgrades/social_post_sharing.js | 23 ++++ src/upgrades/theme_to_active_plugins.js | 18 +++ src/upgrades/upload_privileges.js | 40 +++++++ src/upgrades/user_best_posts.js | 31 +++++ src/upgrades/user_post_count_per_tid.js | 50 ++++++++ src/upgrades/users_notvalidated.js | 31 +++++ 16 files changed, 700 insertions(+), 2 deletions(-) create mode 100644 src/upgrades/assign_topic_read_privilege.js create mode 100644 src/upgrades/chat_room_hashes.js create mode 100644 src/upgrades/chat_upgrade.js create mode 100644 src/upgrades/dismiss_flags_from_deleted_topics.js create mode 100644 src/upgrades/edit_delete_deletetopic_privileges.js create mode 100644 src/upgrades/global_moderators.js create mode 100644 src/upgrades/group_title_update.js create mode 100644 src/upgrades/remove_negative_best_posts.js create mode 100644 src/upgrades/separate_upvote_downvote.js create mode 100644 src/upgrades/social_post_sharing.js create mode 100644 src/upgrades/theme_to_active_plugins.js create mode 100644 src/upgrades/upload_privileges.js create mode 100644 src/upgrades/user_best_posts.js create mode 100644 src/upgrades/user_post_count_per_tid.js create mode 100644 src/upgrades/users_notvalidated.js diff --git a/src/upgrade.js b/src/upgrade.js index 3558a2b30a..33aaa831ca 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -10,9 +10,21 @@ var utils = require('../public/src/utils'); var Upgrade = { available: [ + { + version: '1.0.0', + upgrades: ['chat_upgrade', 'chat_room_hashes', 'theme_to_active_plugins', 'user_best_posts', 'users_notvalidated', 'global_moderators', 'social_post_sharing'], + }, + { + version: '1.1.0', + upgrades: ['group_title_update', 'user_post_count_per_tid', 'dismiss_flags_from_deleted_topics', 'assign_topic_read_privilege', 'separate_upvote_downvote'], + }, + { + version: '1.1.1', + upgrades: ['upload_privileges', 'remove_negative_best_posts'], + }, { version: '1.2.0', - upgrades: ['category_recent_tids'], + upgrades: ['category_recent_tids', 'edit_delete_deletetopic_privileges'], }, { version: '1.3.0', @@ -85,7 +97,7 @@ Upgrade.process = function (files, skipCount, callback) { var scriptExport = require(file); var date = new Date(scriptExport.timestamp); - process.stdout.write(' → '.white + String('[' + [date.getFullYear(), date.getMonth() + 1, date.getDate() + 1].join('/') + '] ').gray + String(scriptExport.name).reset + '... '); + process.stdout.write(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '... '); // Do the upgrade... scriptExport.method(function (err) { diff --git a/src/upgrades/assign_topic_read_privilege.js b/src/upgrades/assign_topic_read_privilege.js new file mode 100644 index 0000000000..1e65d0d6f7 --- /dev/null +++ b/src/upgrades/assign_topic_read_privilege.js @@ -0,0 +1,73 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Giving topics:read privs to any group that was previously allowed to Find & Access Category', + timestamp: Date.UTC(2016, 4, 28), + method: function (callback) { + var groupsAPI = require('../groups'); + var privilegesAPI = require('../privileges'); + + db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) { + if (err) { + return callback(err); + } + + async.eachSeries(cids, function (cid, next) { + privilegesAPI.categories.list(cid, function (err, data) { + if (err) { + return next(err); + } + + var groups = data.groups; + var users = data.users; + + async.waterfall([ + function (next) { + async.eachSeries(groups, function (group, next) { + if (group.privileges['groups:read']) { + return groupsAPI.join('cid:' + cid + ':privileges:groups:topics:read', group.name, function (err) { + if (!err) { + winston.verbose('cid:' + cid + ':privileges:groups:topics:read granted to gid: ' + group.name); + } + + return next(err); + }); + } + + next(null); + }, next); + }, + function (next) { + async.eachSeries(users, function (user, next) { + if (user.privileges.read) { + return groupsAPI.join('cid:' + cid + ':privileges:topics:read', user.uid, function (err) { + if (!err) { + winston.verbose('cid:' + cid + ':privileges:topics:read granted to uid: ' + user.uid); + } + + return next(err); + }); + } + + next(null); + }, next); + }, + ], function (err) { + if (!err) { + winston.verbose('-- cid ' + cid + ' upgraded'); + } + + next(err); + }); + }); + }, callback); + }); + }, +}; diff --git a/src/upgrades/chat_room_hashes.js b/src/upgrades/chat_room_hashes.js new file mode 100644 index 0000000000..bccf85282c --- /dev/null +++ b/src/upgrades/chat_room_hashes.js @@ -0,0 +1,41 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); + +module.exports = { + name: 'Chat room hashes', + timestamp: Date.UTC(2015, 11, 23), + method: function (callback) { + db.getObjectField('global', 'nextChatRoomId', function (err, nextChatRoomId) { + if (err) { + return callback(err); + } + var currentChatRoomId = 1; + async.whilst(function () { + return currentChatRoomId <= nextChatRoomId; + }, function (next) { + db.getSortedSetRange('chat:room:' + currentChatRoomId + ':uids', 0, 0, function (err, uids) { + if (err) { + return next(err); + } + if (!Array.isArray(uids) || !uids.length || !uids[0]) { + currentChatRoomId += 1; + return next(); + } + + db.setObject('chat:room:' + currentChatRoomId, { owner: uids[0], roomId: currentChatRoomId }, function (err) { + if (err) { + return next(err); + } + currentChatRoomId += 1; + next(); + }); + }); + }, callback); + }); + }, +}; diff --git a/src/upgrades/chat_upgrade.js b/src/upgrades/chat_upgrade.js new file mode 100644 index 0000000000..ce49b501de --- /dev/null +++ b/src/upgrades/chat_upgrade.js @@ -0,0 +1,87 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Upgrading chats', + timestamp: Date.UTC(2015, 11, 15), + method: function (callback) { + db.getObjectFields('global', ['nextMid', 'nextChatRoomId'], function (err, globalData) { + if (err) { + return callback(err); + } + + var rooms = {}; + var roomId = globalData.nextChatRoomId || 1; + var currentMid = 1; + + async.whilst(function () { + return currentMid <= globalData.nextMid; + }, function (next) { + db.getObject('message:' + currentMid, function (err, message) { + var msgTime; + + function addMessageToUids(roomId, callback) { + async.parallel([ + function (next) { + db.sortedSetAdd('uid:' + message.fromuid + ':chat:room:' + roomId + ':mids', msgTime, currentMid, next); + }, + function (next) { + db.sortedSetAdd('uid:' + message.touid + ':chat:room:' + roomId + ':mids', msgTime, currentMid, next); + }, + ], callback); + } + + if (err || !message) { + winston.verbose('skipping chat message ', currentMid); + currentMid += 1; + return next(err); + } + + var pairID = [parseInt(message.fromuid, 10), parseInt(message.touid, 10)].sort().join(':'); + msgTime = parseInt(message.timestamp, 10); + + if (rooms[pairID]) { + winston.verbose('adding message ' + currentMid + ' to existing roomID ' + roomId); + addMessageToUids(rooms[pairID], function (err) { + if (err) { + return next(err); + } + currentMid += 1; + next(); + }); + } else { + winston.verbose('adding message ' + currentMid + ' to new roomID ' + roomId); + async.parallel([ + function (next) { + db.sortedSetAdd('uid:' + message.fromuid + ':chat:rooms', msgTime, roomId, next); + }, + function (next) { + db.sortedSetAdd('uid:' + message.touid + ':chat:rooms', msgTime, roomId, next); + }, + function (next) { + db.sortedSetAdd('chat:room:' + roomId + ':uids', [msgTime, msgTime + 1], [message.fromuid, message.touid], next); + }, + function (next) { + addMessageToUids(roomId, next); + }, + ], function (err) { + if (err) { + return next(err); + } + rooms[pairID] = roomId; + roomId += 1; + currentMid += 1; + db.setObjectField('global', 'nextChatRoomId', roomId, next); + }); + } + }); + }, callback); + }); + }, +}; diff --git a/src/upgrades/dismiss_flags_from_deleted_topics.js b/src/upgrades/dismiss_flags_from_deleted_topics.js new file mode 100644 index 0000000000..cf4af73436 --- /dev/null +++ b/src/upgrades/dismiss_flags_from_deleted_topics.js @@ -0,0 +1,43 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Dismiss flags from deleted topics', + timestamp: Date.UTC(2016, 3, 29), + method: function (callback) { + var posts = require('../posts'); + var topics = require('../topics'); + + var pids; + var tids; + + async.waterfall([ + async.apply(db.getSortedSetRange, 'posts:flagged', 0, -1), + function (_pids, next) { + pids = _pids; + posts.getPostsFields(pids, ['tid'], next); + }, + function (_tids, next) { + tids = _tids.map(function (a) { + return a.tid; + }); + + topics.getTopicsFields(tids, ['deleted'], next); + }, + function (state, next) { + var toDismiss = state.map(function (a, idx) { + return parseInt(a.deleted, 10) === 1 ? pids[idx] : null; + }).filter(Boolean); + + winston.verbose('[2016/04/29] ' + toDismiss.length + ' dismissable flags found'); + async.each(toDismiss, posts.dismissFlag, next); + }, + ], callback); + }, +}; diff --git a/src/upgrades/edit_delete_deletetopic_privileges.js b/src/upgrades/edit_delete_deletetopic_privileges.js new file mode 100644 index 0000000000..5ba6f604c5 --- /dev/null +++ b/src/upgrades/edit_delete_deletetopic_privileges.js @@ -0,0 +1,109 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Granting edit/delete/delete topic on existing categories', + timestamp: Date.UTC(2016, 7, 7), + method: function (callback) { + var groupsAPI = require('../groups'); + var privilegesAPI = require('../privileges'); + + db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) { + if (err) { + return callback(err); + } + + async.eachSeries(cids, function (cid, next) { + privilegesAPI.categories.list(cid, function (err, data) { + if (err) { + return next(err); + } + + var groups = data.groups; + var users = data.users; + + async.waterfall([ + function (next) { + async.eachSeries(groups, function (group, next) { + if (group.privileges['groups:topics:reply']) { + return async.parallel([ + async.apply(groupsAPI.join, 'cid:' + cid + ':privileges:groups:posts:edit', group.name), + async.apply(groupsAPI.join, 'cid:' + cid + ':privileges:groups:posts:delete', group.name), + ], function (err) { + if (!err) { + winston.verbose('cid:' + cid + ':privileges:groups:posts:edit, cid:' + cid + ':privileges:groups:posts:delete granted to gid: ' + group.name); + } + + return next(err); + }); + } + + next(null); + }, next); + }, + function (next) { + async.eachSeries(groups, function (group, next) { + if (group.privileges['groups:topics:create']) { + return groupsAPI.join('cid:' + cid + ':privileges:groups:topics:delete', group.name, function (err) { + if (!err) { + winston.verbose('cid:' + cid + ':privileges:groups:topics:delete granted to gid: ' + group.name); + } + + return next(err); + }); + } + + next(null); + }, next); + }, + function (next) { + async.eachSeries(users, function (user, next) { + if (user.privileges['topics:reply']) { + return async.parallel([ + async.apply(groupsAPI.join, 'cid:' + cid + ':privileges:posts:edit', user.uid), + async.apply(groupsAPI.join, 'cid:' + cid + ':privileges:posts:delete', user.uid), + ], function (err) { + if (!err) { + winston.verbose('cid:' + cid + ':privileges:posts:edit, cid:' + cid + ':privileges:posts:delete granted to uid: ' + user.uid); + } + + return next(err); + }); + } + + next(null); + }, next); + }, + function (next) { + async.eachSeries(users, function (user, next) { + if (user.privileges['topics:create']) { + return groupsAPI.join('cid:' + cid + ':privileges:topics:delete', user.uid, function (err) { + if (!err) { + winston.verbose('cid:' + cid + ':privileges:topics:delete granted to uid: ' + user.uid); + } + + return next(err); + }); + } + + next(null); + }, next); + }, + ], function (err) { + if (!err) { + winston.verbose('-- cid ' + cid + ' upgraded'); + } + + next(err); + }); + }); + }, callback); + }); + }, +}; diff --git a/src/upgrades/global_moderators.js b/src/upgrades/global_moderators.js new file mode 100644 index 0000000000..bfe909a11b --- /dev/null +++ b/src/upgrades/global_moderators.js @@ -0,0 +1,34 @@ +/* jslint node: true */ + +'use strict'; + +var async = require('async'); + +module.exports = { + name: 'Creating Global moderators group', + timestamp: Date.UTC(2016, 0, 23), + method: function (callback) { + var groups = require('../groups'); + async.waterfall([ + function (next) { + groups.exists('Global Moderators', next); + }, + function (exists, next) { + if (exists) { + return next(null, null); + } + groups.create({ + name: 'Global Moderators', + userTitle: 'Global Moderator', + description: 'Forum wide moderators', + hidden: 0, + private: 1, + disableJoinRequests: 1, + }, next); + }, + function (groupData, next) { + groups.show('Global Moderators', next); + }, + ], callback); + }, +}; diff --git a/src/upgrades/group_title_update.js b/src/upgrades/group_title_update.js new file mode 100644 index 0000000000..5f82e97bfe --- /dev/null +++ b/src/upgrades/group_title_update.js @@ -0,0 +1,34 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Group title from settings to user profile', + timestamp: Date.UTC(2016, 3, 14), + method: function (callback) { + var user = require('../user'); + var batch = require('../batch'); + var count = 0; + batch.processSortedSet('users:joindate', function (uids, next) { + winston.verbose('upgraded ' + count + ' users'); + user.getMultipleUserSettings(uids, function (err, settings) { + if (err) { + return next(err); + } + count += uids.length; + settings = settings.filter(function (setting) { + return setting && setting.groupTitle; + }); + + async.each(settings, function (setting, next) { + db.setObjectField('user:' + setting.uid, 'groupTitle', setting.groupTitle, next); + }, next); + }); + }, {}, callback); + }, +}; diff --git a/src/upgrades/remove_negative_best_posts.js b/src/upgrades/remove_negative_best_posts.js new file mode 100644 index 0000000000..604d1d8234 --- /dev/null +++ b/src/upgrades/remove_negative_best_posts.js @@ -0,0 +1,22 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Removing best posts with negative scores', + timestamp: Date.UTC(2016, 7, 5), + method: function (callback) { + var batch = require('../batch'); + batch.processSortedSet('users:joindate', function (ids, next) { + async.each(ids, function (id, next) { + winston.verbose('processing uid ' + id); + db.sortedSetsRemoveRangeByScore(['uid:' + id + ':posts:votes'], '-inf', 0, next); + }, next); + }, {}, callback); + }, +}; diff --git a/src/upgrades/separate_upvote_downvote.js b/src/upgrades/separate_upvote_downvote.js new file mode 100644 index 0000000000..b313c7439d --- /dev/null +++ b/src/upgrades/separate_upvote_downvote.js @@ -0,0 +1,50 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Store upvotes/downvotes separately', + timestamp: Date.UTC(2016, 5, 13), + method: function (callback) { + var batch = require('../batch'); + var posts = require('../posts'); + var count = 0; + batch.processSortedSet('posts:pid', function (pids, next) { + winston.verbose('upgraded ' + count + ' posts'); + count += pids.length; + async.each(pids, function (pid, next) { + async.parallel({ + upvotes: function (next) { + db.setCount('pid:' + pid + ':upvote', next); + }, + downvotes: function (next) { + db.setCount('pid:' + pid + ':downvote', next); + }, + }, function (err, results) { + if (err) { + return next(err); + } + var data = {}; + + if (parseInt(results.upvotes, 10) > 0) { + data.upvotes = results.upvotes; + } + if (parseInt(results.downvotes, 10) > 0) { + data.downvotes = results.downvotes; + } + + if (Object.keys(data).length) { + posts.setPostFields(pid, data, next); + } else { + next(); + } + }, next); + }, next); + }, {}, callback); + }, +}; diff --git a/src/upgrades/social_post_sharing.js b/src/upgrades/social_post_sharing.js new file mode 100644 index 0000000000..22c4a8c7cf --- /dev/null +++ b/src/upgrades/social_post_sharing.js @@ -0,0 +1,23 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); + +module.exports = { + name: 'Social: Post Sharing', + timestamp: Date.UTC(2016, 1, 25), + method: function (callback) { + var social = require('../social'); + async.parallel([ + function (next) { + social.setActivePostSharingNetworks(['facebook', 'google', 'twitter'], next); + }, + function (next) { + db.deleteObjectField('config', 'disableSocialButtons', next); + }, + ], callback); + }, +}; diff --git a/src/upgrades/theme_to_active_plugins.js b/src/upgrades/theme_to_active_plugins.js new file mode 100644 index 0000000000..3adbca16dc --- /dev/null +++ b/src/upgrades/theme_to_active_plugins.js @@ -0,0 +1,18 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); + +module.exports = { + name: 'Adding theme to active plugins sorted set', + timestamp: Date.UTC(2015, 11, 23), + method: function (callback) { + async.waterfall([ + async.apply(db.getObjectField, 'config', 'theme:id'), + async.apply(db.sortedSetAdd, 'plugins:active', 0), + ], callback); + }, +}; diff --git a/src/upgrades/upload_privileges.js b/src/upgrades/upload_privileges.js new file mode 100644 index 0000000000..b3268a2022 --- /dev/null +++ b/src/upgrades/upload_privileges.js @@ -0,0 +1,40 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); + +module.exports = { + name: 'Giving upload privileges', + timestamp: Date.UTC(2016, 6, 12), + method: function (callback) { + var privilegesAPI = require('../privileges'); + var meta = require('../meta'); + + db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) { + if (err) { + return callback(err); + } + + async.eachSeries(cids, function (cid, next) { + privilegesAPI.categories.list(cid, function (err, data) { + if (err) { + return next(err); + } + async.eachSeries(data.groups, function (group, next) { + if (group.name === 'guests' && parseInt(meta.config.allowGuestUploads, 10) !== 1) { + return next(); + } + if (group.privileges['groups:read']) { + privilegesAPI.categories.give(['upload:post:image'], cid, group.name, next); + } else { + next(); + } + }, next); + }); + }, callback); + }); + }, +}; diff --git a/src/upgrades/user_best_posts.js b/src/upgrades/user_best_posts.js new file mode 100644 index 0000000000..126539c2b7 --- /dev/null +++ b/src/upgrades/user_best_posts.js @@ -0,0 +1,31 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Creating user best post sorted sets', + timestamp: Date.UTC(2016, 0, 14), + method: function (callback) { + var batch = require('../batch'); + + batch.processSortedSet('posts:pid', function (ids, next) { + async.eachSeries(ids, function (id, next) { + db.getObjectFields('post:' + id, ['pid', 'uid', 'votes'], function (err, postData) { + if (err) { + return next(err); + } + if (!postData || !parseInt(postData.votes, 10) || !parseInt(postData.uid, 10)) { + return next(); + } + winston.verbose('processing pid: ' + postData.pid + ' uid: ' + postData.uid + ' votes: ' + postData.votes); + db.sortedSetAdd('uid:' + postData.uid + ':posts:votes', postData.votes, postData.pid, next); + }); + }, next); + }, {}, callback); + }, +}; diff --git a/src/upgrades/user_post_count_per_tid.js b/src/upgrades/user_post_count_per_tid.js new file mode 100644 index 0000000000..c0e86c4f52 --- /dev/null +++ b/src/upgrades/user_post_count_per_tid.js @@ -0,0 +1,50 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Users post count per tid', + timestamp: Date.UTC(2016, 3, 19), + method: function (callback) { + var batch = require('../batch'); + var topics = require('../topics'); + var count = 0; + batch.processSortedSet('topics:tid', function (tids, next) { + winston.verbose('upgraded ' + count + ' topics'); + count += tids.length; + async.each(tids, function (tid, next) { + db.delete('tid:' + tid + ':posters', function (err) { + if (err) { + return next(err); + } + topics.getPids(tid, function (err, pids) { + if (err) { + return next(err); + } + + if (!pids.length) { + return next(); + } + + async.eachSeries(pids, function (pid, next) { + db.getObjectField('post:' + pid, 'uid', function (err, uid) { + if (err) { + return next(err); + } + if (!parseInt(uid, 10)) { + return next(); + } + db.sortedSetIncrBy('tid:' + tid + ':posters', 1, uid, next); + }); + }, next); + }); + }); + }, next); + }, {}, callback); + }, +}; diff --git a/src/upgrades/users_notvalidated.js b/src/upgrades/users_notvalidated.js new file mode 100644 index 0000000000..9c23b6a539 --- /dev/null +++ b/src/upgrades/users_notvalidated.js @@ -0,0 +1,31 @@ +/* jslint node: true */ + +'use strict'; + +var db = require('../database'); + +var async = require('async'); +var winston = require('winston'); + +module.exports = { + name: 'Creating users:notvalidated', + timestamp: Date.UTC(2016, 0, 20), + method: function (callback) { + var batch = require('../batch'); + var now = Date.now(); + batch.processSortedSet('users:joindate', function (ids, next) { + async.eachSeries(ids, function (id, next) { + db.getObjectFields('user:' + id, ['uid', 'email:confirmed'], function (err, userData) { + if (err) { + return next(err); + } + if (!userData || !parseInt(userData.uid, 10) || parseInt(userData['email:confirmed'], 10) === 1) { + return next(); + } + winston.verbose('processing uid: ' + userData.uid + ' email:confirmed: ' + userData['email:confirmed']); + db.sortedSetAdd('users:notvalidated', now, userData.uid, next); + }); + }, next); + }, callback); + }, +}; From b8ee09aacb4fe0a924d2175e7136cabfd38ec655 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sat, 25 Feb 2017 15:45:45 -0500 Subject: [PATCH 06/16] re: #5467 -- requiring exact filename match sans extension, for a single upgrade to work --- src/upgrade.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/upgrade.js b/src/upgrade.js index 33aaa831ca..a5e4d89158 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -75,7 +75,7 @@ Upgrade.runSingle = function (query, callback) { async.apply(utils.walk, path.join(__dirname, './upgrades')), function (files, next) { next(null, files.filter(function (file) { - return file.search(new RegExp(query)) !== -1; + return path.basename(file, '.js') === query; })); }, ], function (err, files) { From 3b1b2d39c6a03f846f040a9d779ad37fc809c651 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sat, 25 Feb 2017 21:43:29 -0500 Subject: [PATCH 07/16] added some instructions to upgrade script file --- src/upgrade.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/upgrade.js b/src/upgrade.js index a5e4d89158..06b4ec50ed 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -8,6 +8,16 @@ var path = require('path'); var db = require('./database'); var utils = require('../public/src/utils'); +/* + * Need to write an upgrade script for NodeBB? Cool. + * + * 1. Copy TEMPLATE to a file name of your choice. Try to be succinct. + * 2. Open up that file and change the user-friendly name (can be longer/more descriptive than the file name) + * and timestamp + * 3. Add your script under the "method" property + * 4. Append your filename to the array below for the next NodeBB version. + */ + var Upgrade = { available: [ { @@ -35,7 +45,7 @@ var Upgrade = { upgrades: ['global_and_user_language_keys', 'sorted_set_for_pinned_topics'], }, { - version: '1.5.0', + version: 'develop', // rename this to whatever the next NodeBB version is upgrades: ['flags_refactor'], }, ], From d75cc60e7644a755f784126b77d7b7befc2139d3 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sun, 26 Feb 2017 15:53:25 -0500 Subject: [PATCH 08/16] added upgrade.check back into upgrade.js --- src/start.js | 12 ++++++------ src/upgrade.js | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/start.js b/src/start.js index 87c0d2bf70..731e939911 100644 --- a/src/start.js +++ b/src/start.js @@ -58,16 +58,16 @@ start.start = function () { if (err) { switch (err.message) { case 'schema-out-of-date': - winston.warn('Your NodeBB schema is out-of-date. Please run the following command to bring your dataset up to spec:'); - winston.warn(' ./nodebb upgrade'); + winston.error('Your NodeBB schema is out-of-date. Please run the following command to bring your dataset up to spec:'); + winston.error(' ./nodebb upgrade'); break; case 'dependencies-out-of-date': - winston.warn('One or more of NodeBB\'s dependent packages are out-of-date. Please run the following command to update them:'); - winston.warn(' ./nodebb upgrade'); + winston.error('One or more of NodeBB\'s dependent packages are out-of-date. Please run the following command to update them:'); + winston.error(' ./nodebb upgrade'); break; case 'dependencies-missing': - winston.warn('One or more of NodeBB\'s dependent packages are missing. Please run the following command to update them:'); - winston.warn(' ./nodebb upgrade'); + winston.error('One or more of NodeBB\'s dependent packages are missing. Please run the following command to update them:'); + winston.error(' ./nodebb upgrade'); break; default: winston.error(err); diff --git a/src/upgrade.js b/src/upgrade.js index 06b4ec50ed..0f7e55c438 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -51,6 +51,22 @@ var Upgrade = { ], }; +Upgrade.check = function (callback) { + // Throw 'schema-out-of-date' if not all upgrade scripts have run + var all = Upgrade.available.reduce(function (memo, current) { + memo = memo.concat(current.upgrades); + return memo; + }, []); + + db.getSortedSetRange('schemaLog', 0, -1, function (err, executed) { + var remainder = all.filter(function (name) { + return executed.indexOf(name) === -1; + }); + + callback(remainder.length > 1 ? new Error('schema-out-of-date') : null); + }); +}; + Upgrade.run = function (callback) { process.stdout.write('\nParsing upgrade scripts... '); var queue = []; From a2662f8b696918f52a712a0623fe0b2a55b13573 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sun, 26 Feb 2017 16:03:26 -0500 Subject: [PATCH 09/16] linting for tests to pass --- src/upgrade.js | 4 ++++ src/upgrades/category_recent_tids.js | 6 +++--- src/upgrades/favourites_to_bookmarks.js | 8 ++++---- src/upgrades/flags_refactor.js | 14 +++++++------- src/upgrades/global_and_user_language_keys.js | 14 +++++--------- src/upgrades/sorted_set_for_pinned_topics.js | 7 ++++--- src/upgrades/sorted_sets_for_post_replies.js | 7 ++++--- 7 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/upgrade.js b/src/upgrade.js index 0f7e55c438..00f599fbdb 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -59,6 +59,10 @@ Upgrade.check = function (callback) { }, []); db.getSortedSetRange('schemaLog', 0, -1, function (err, executed) { + if (err) { + return callback(err); + } + var remainder = all.filter(function (name) { return executed.indexOf(name) === -1; }); diff --git a/src/upgrades/category_recent_tids.js b/src/upgrades/category_recent_tids.js index ccb6f5c818..18d27266f6 100644 --- a/src/upgrades/category_recent_tids.js +++ b/src/upgrades/category_recent_tids.js @@ -1,10 +1,10 @@ /* jslint node: true */ + 'use strict'; var db = require('../database'); var async = require('async'); -var winston = require('winston'); module.exports = { name: 'Category recent tids', @@ -29,5 +29,5 @@ module.exports = { }); }, callback); }); - } -}; \ No newline at end of file + }, +}; diff --git a/src/upgrades/favourites_to_bookmarks.js b/src/upgrades/favourites_to_bookmarks.js index 288d32c511..05b1e5b5f9 100644 --- a/src/upgrades/favourites_to_bookmarks.js +++ b/src/upgrades/favourites_to_bookmarks.js @@ -1,10 +1,10 @@ /* jslint node: true */ + 'use strict'; var db = require('../database'); var async = require('async'); -var winston = require('winston'); module.exports = { name: 'Favourites to Bookmarks', @@ -31,7 +31,7 @@ module.exports = { }, function (next) { db.deleteObjectField('post:' + id, 'reputation', next); - } + }, ], next); }, next); }, {}, next); @@ -48,5 +48,5 @@ module.exports = { } async.series([upgradePosts, upgradeUsers], callback); - } -}; \ No newline at end of file + }, +}; diff --git a/src/upgrades/flags_refactor.js b/src/upgrades/flags_refactor.js index c61a3a4747..deb1befc6c 100644 --- a/src/upgrades/flags_refactor.js +++ b/src/upgrades/flags_refactor.js @@ -1,4 +1,5 @@ /* jslint node: true */ + 'use strict'; var db = require('../database'); @@ -12,7 +13,6 @@ module.exports = { var batch = require('../batch'); var posts = require('../posts'); var flags = require('../flags'); - var migrated = 0; batch.processSortedSet('posts:pid', function (ids, next) { posts.getPostsByPids(ids, 1, function (err, posts) { @@ -23,11 +23,11 @@ module.exports = { posts = posts.filter(function (post) { return post.hasOwnProperty('flags'); }); - + async.each(posts, function (post, next) { async.parallel({ uids: async.apply(db.getSortedSetRangeWithScores, 'pid:' + post.pid + ':flag:uids', 0, -1), - reasons: async.apply(db.getSortedSetRange, 'pid:' + post.pid + ':flag:uid:reason', 0, -1) + reasons: async.apply(db.getSortedSetRange, 'pid:' + post.pid + ':flag:uid:reason', 0, -1), }, function (err, data) { if (err) { return next(err); @@ -51,7 +51,7 @@ module.exports = { flags.update(flagObj.flagId, 1, { state: post['flag:state'], assignee: post['flag:assignee'], - datetime: datetime + datetime: datetime, }, next); } else { setImmediate(next); @@ -72,7 +72,7 @@ module.exports = { } else { setImmediate(next); } - } + }, ], function (err) { if (err && err.message === '[[error:already-flagged]]') { // Already flagged, no need to parse, but not an error @@ -85,5 +85,5 @@ module.exports = { }, next); }); }, callback); - } -}; \ No newline at end of file + }, +}; diff --git a/src/upgrades/global_and_user_language_keys.js b/src/upgrades/global_and_user_language_keys.js index d08b3fea74..14f71c1fce 100644 --- a/src/upgrades/global_and_user_language_keys.js +++ b/src/upgrades/global_and_user_language_keys.js @@ -1,10 +1,10 @@ /* jslint node: true */ + 'use strict'; var db = require('../database'); var async = require('async'); -var winston = require('winston'); module.exports = { name: 'Update global and user language keys', @@ -14,8 +14,6 @@ module.exports = { var meta = require('../meta'); var batch = require('../batch'); var newLanguage; - var i = 0; - var j = 0; async.parallel([ function (next) { meta.configs.get('defaultLang', function (err, defaultLang) { @@ -41,23 +39,21 @@ module.exports = { async.waterfall([ async.apply(db.getObjectField, 'user:' + uid + ':settings', 'userLang'), function (language, next) { - ++i; if (!language) { return setImmediate(next); } newLanguage = language.replace('_', '-').replace('@', '-x-'); if (newLanguage !== language) { - ++j; user.setSetting(uid, 'userLang', newLanguage, next); } else { setImmediate(next); } - } + }, ], next); }, next); }, next); - } + }, ], callback); - } -}; \ No newline at end of file + }, +}; diff --git a/src/upgrades/sorted_set_for_pinned_topics.js b/src/upgrades/sorted_set_for_pinned_topics.js index d10a6ee902..faa12f3cc8 100644 --- a/src/upgrades/sorted_set_for_pinned_topics.js +++ b/src/upgrades/sorted_set_for_pinned_topics.js @@ -1,4 +1,5 @@ /* jslint node: true */ + 'use strict'; var db = require('../database'); @@ -28,10 +29,10 @@ module.exports = { async.parallel([ async.apply(db.sortedSetAdd, 'cid:' + topicData.cid + ':tids:pinned', Date.now(), topicData.tid), async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids', topicData.tid), - async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids:posts', topicData.tid) + async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids:posts', topicData.tid), ], next); }, next); }); }, callback); - } -}; \ No newline at end of file + }, +}; diff --git a/src/upgrades/sorted_sets_for_post_replies.js b/src/upgrades/sorted_sets_for_post_replies.js index 7442b614d4..78fc96c7a1 100644 --- a/src/upgrades/sorted_sets_for_post_replies.js +++ b/src/upgrades/sorted_sets_for_post_replies.js @@ -1,4 +1,5 @@ /* jslint node: true */ + 'use strict'; var db = require('../database'); @@ -25,10 +26,10 @@ module.exports = { winston.verbose('processing pid: ' + postData.pid + ' toPid: ' + postData.toPid); async.parallel([ async.apply(db.sortedSetAdd, 'pid:' + postData.toPid + ':replies', postData.timestamp, postData.pid), - async.apply(db.incrObjectField, 'post:' + postData.toPid, 'replies') + async.apply(db.incrObjectField, 'post:' + postData.toPid, 'replies'), ], next); }, next); }); }, callback); - } -}; \ No newline at end of file + }, +}; From 93c96da0b9433a90dadd2ac5b582c2abe454ab0f Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sun, 26 Feb 2017 16:26:09 -0500 Subject: [PATCH 10/16] adjusted upgrade check logic for instances with no schemaLog --- src/install.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/install.js b/src/install.js index a06a138f26..e3abfdca2f 100644 --- a/src/install.js +++ b/src/install.js @@ -507,12 +507,11 @@ install.setup = function (callback) { setCopyrightWidget, function (next) { var upgrade = require('./upgrade'); - upgrade.check(function (err, uptodate) { - if (err) { + upgrade.check(function (err) { + if (err && err.message === 'schema-out-of-date') { + upgrade.run(next); + } else if (err) { return next(err); - } - if (!uptodate) { - upgrade.upgrade(next); } else { next(); } From 8f5d1ca4da68c6e23f2fe41a801bf98a4cf1b50a Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 27 Feb 2017 10:15:44 -0500 Subject: [PATCH 11/16] added null check for #5482 --- src/upgrades/sound_settings.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/upgrades/sound_settings.js b/src/upgrades/sound_settings.js index a31a573808..f925a1ea2e 100644 --- a/src/upgrades/sound_settings.js +++ b/src/upgrades/sound_settings.js @@ -25,7 +25,7 @@ module.exports = { var keys = ['chat-incoming', 'chat-outgoing', 'notification']; db.getObject('settings:sounds', function (err, settings) { - if (err) { + if (err || !settings) { return cb(err); } From a8dd7946979f9cf12e0f1b4cd75fc7f1ee2afb72 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Tue, 28 Feb 2017 14:04:44 -0500 Subject: [PATCH 12/16] reapply ef93ef3dd4e26803e3c19a2c8a3268fb3e15d9e7 --- src/upgrades/sound_settings.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/upgrades/sound_settings.js b/src/upgrades/sound_settings.js index 4c8359fc2e..5ba958f591 100644 --- a/src/upgrades/sound_settings.js +++ b/src/upgrades/sound_settings.js @@ -42,8 +42,8 @@ module.exports = { batch.processSortedSet('users:joindate', function (ids, next) { async.each(ids, function (uid, next) { - user.getSettings(uid, function (err, settings) { - if (err) { + db.getObject('user:' + uid + ':settings', function (err, settings) { + if (err || !settings) { return next(err); } From fc13776f26e93e9f4cbd33d168612e79ace638f8 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 10 Mar 2017 16:36:22 -0500 Subject: [PATCH 13/16] moving files from src/upgrades to individual folders by version --- src/upgrades/{ => 1.0.0}/chat_room_hashes.js | 0 src/upgrades/{ => 1.0.0}/chat_upgrade.js | 0 src/upgrades/{ => 1.0.0}/global_moderators.js | 0 src/upgrades/{ => 1.0.0}/social_post_sharing.js | 0 src/upgrades/{ => 1.0.0}/theme_to_active_plugins.js | 0 src/upgrades/{ => 1.0.0}/user_best_posts.js | 0 src/upgrades/{ => 1.0.0}/users_notvalidated.js | 0 src/upgrades/{ => 1.1.0}/assign_topic_read_privilege.js | 0 src/upgrades/{ => 1.1.0}/dismiss_flags_from_deleted_topics.js | 0 src/upgrades/{ => 1.1.0}/group_title_update.js | 0 src/upgrades/{ => 1.1.0}/separate_upvote_downvote.js | 0 src/upgrades/{ => 1.1.0}/user_post_count_per_tid.js | 0 src/upgrades/{ => 1.1.1}/remove_negative_best_posts.js | 0 src/upgrades/{ => 1.1.1}/upload_privileges.js | 0 src/upgrades/{ => 1.2.0}/category_recent_tids.js | 0 src/upgrades/{ => 1.2.0}/edit_delete_deletetopic_privileges.js | 0 src/upgrades/{ => 1.3.0}/favourites_to_bookmarks.js | 0 src/upgrades/{ => 1.3.0}/sorted_sets_for_post_replies.js | 0 src/upgrades/{ => 1.4.0}/global_and_user_language_keys.js | 0 src/upgrades/{ => 1.4.0}/sorted_set_for_pinned_topics.js | 0 src/upgrades/{ => develop}/flags_refactor.js | 0 src/upgrades/{ => develop}/post_votes_zset.js | 0 src/upgrades/{ => master}/config_urls_update.js | 0 src/upgrades/{ => master}/sound_settings.js | 0 24 files changed, 0 insertions(+), 0 deletions(-) rename src/upgrades/{ => 1.0.0}/chat_room_hashes.js (100%) rename src/upgrades/{ => 1.0.0}/chat_upgrade.js (100%) rename src/upgrades/{ => 1.0.0}/global_moderators.js (100%) rename src/upgrades/{ => 1.0.0}/social_post_sharing.js (100%) rename src/upgrades/{ => 1.0.0}/theme_to_active_plugins.js (100%) rename src/upgrades/{ => 1.0.0}/user_best_posts.js (100%) rename src/upgrades/{ => 1.0.0}/users_notvalidated.js (100%) rename src/upgrades/{ => 1.1.0}/assign_topic_read_privilege.js (100%) rename src/upgrades/{ => 1.1.0}/dismiss_flags_from_deleted_topics.js (100%) rename src/upgrades/{ => 1.1.0}/group_title_update.js (100%) rename src/upgrades/{ => 1.1.0}/separate_upvote_downvote.js (100%) rename src/upgrades/{ => 1.1.0}/user_post_count_per_tid.js (100%) rename src/upgrades/{ => 1.1.1}/remove_negative_best_posts.js (100%) rename src/upgrades/{ => 1.1.1}/upload_privileges.js (100%) rename src/upgrades/{ => 1.2.0}/category_recent_tids.js (100%) rename src/upgrades/{ => 1.2.0}/edit_delete_deletetopic_privileges.js (100%) rename src/upgrades/{ => 1.3.0}/favourites_to_bookmarks.js (100%) rename src/upgrades/{ => 1.3.0}/sorted_sets_for_post_replies.js (100%) rename src/upgrades/{ => 1.4.0}/global_and_user_language_keys.js (100%) rename src/upgrades/{ => 1.4.0}/sorted_set_for_pinned_topics.js (100%) rename src/upgrades/{ => develop}/flags_refactor.js (100%) rename src/upgrades/{ => develop}/post_votes_zset.js (100%) rename src/upgrades/{ => master}/config_urls_update.js (100%) rename src/upgrades/{ => master}/sound_settings.js (100%) diff --git a/src/upgrades/chat_room_hashes.js b/src/upgrades/1.0.0/chat_room_hashes.js similarity index 100% rename from src/upgrades/chat_room_hashes.js rename to src/upgrades/1.0.0/chat_room_hashes.js diff --git a/src/upgrades/chat_upgrade.js b/src/upgrades/1.0.0/chat_upgrade.js similarity index 100% rename from src/upgrades/chat_upgrade.js rename to src/upgrades/1.0.0/chat_upgrade.js diff --git a/src/upgrades/global_moderators.js b/src/upgrades/1.0.0/global_moderators.js similarity index 100% rename from src/upgrades/global_moderators.js rename to src/upgrades/1.0.0/global_moderators.js diff --git a/src/upgrades/social_post_sharing.js b/src/upgrades/1.0.0/social_post_sharing.js similarity index 100% rename from src/upgrades/social_post_sharing.js rename to src/upgrades/1.0.0/social_post_sharing.js diff --git a/src/upgrades/theme_to_active_plugins.js b/src/upgrades/1.0.0/theme_to_active_plugins.js similarity index 100% rename from src/upgrades/theme_to_active_plugins.js rename to src/upgrades/1.0.0/theme_to_active_plugins.js diff --git a/src/upgrades/user_best_posts.js b/src/upgrades/1.0.0/user_best_posts.js similarity index 100% rename from src/upgrades/user_best_posts.js rename to src/upgrades/1.0.0/user_best_posts.js diff --git a/src/upgrades/users_notvalidated.js b/src/upgrades/1.0.0/users_notvalidated.js similarity index 100% rename from src/upgrades/users_notvalidated.js rename to src/upgrades/1.0.0/users_notvalidated.js diff --git a/src/upgrades/assign_topic_read_privilege.js b/src/upgrades/1.1.0/assign_topic_read_privilege.js similarity index 100% rename from src/upgrades/assign_topic_read_privilege.js rename to src/upgrades/1.1.0/assign_topic_read_privilege.js diff --git a/src/upgrades/dismiss_flags_from_deleted_topics.js b/src/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js similarity index 100% rename from src/upgrades/dismiss_flags_from_deleted_topics.js rename to src/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js diff --git a/src/upgrades/group_title_update.js b/src/upgrades/1.1.0/group_title_update.js similarity index 100% rename from src/upgrades/group_title_update.js rename to src/upgrades/1.1.0/group_title_update.js diff --git a/src/upgrades/separate_upvote_downvote.js b/src/upgrades/1.1.0/separate_upvote_downvote.js similarity index 100% rename from src/upgrades/separate_upvote_downvote.js rename to src/upgrades/1.1.0/separate_upvote_downvote.js diff --git a/src/upgrades/user_post_count_per_tid.js b/src/upgrades/1.1.0/user_post_count_per_tid.js similarity index 100% rename from src/upgrades/user_post_count_per_tid.js rename to src/upgrades/1.1.0/user_post_count_per_tid.js diff --git a/src/upgrades/remove_negative_best_posts.js b/src/upgrades/1.1.1/remove_negative_best_posts.js similarity index 100% rename from src/upgrades/remove_negative_best_posts.js rename to src/upgrades/1.1.1/remove_negative_best_posts.js diff --git a/src/upgrades/upload_privileges.js b/src/upgrades/1.1.1/upload_privileges.js similarity index 100% rename from src/upgrades/upload_privileges.js rename to src/upgrades/1.1.1/upload_privileges.js diff --git a/src/upgrades/category_recent_tids.js b/src/upgrades/1.2.0/category_recent_tids.js similarity index 100% rename from src/upgrades/category_recent_tids.js rename to src/upgrades/1.2.0/category_recent_tids.js diff --git a/src/upgrades/edit_delete_deletetopic_privileges.js b/src/upgrades/1.2.0/edit_delete_deletetopic_privileges.js similarity index 100% rename from src/upgrades/edit_delete_deletetopic_privileges.js rename to src/upgrades/1.2.0/edit_delete_deletetopic_privileges.js diff --git a/src/upgrades/favourites_to_bookmarks.js b/src/upgrades/1.3.0/favourites_to_bookmarks.js similarity index 100% rename from src/upgrades/favourites_to_bookmarks.js rename to src/upgrades/1.3.0/favourites_to_bookmarks.js diff --git a/src/upgrades/sorted_sets_for_post_replies.js b/src/upgrades/1.3.0/sorted_sets_for_post_replies.js similarity index 100% rename from src/upgrades/sorted_sets_for_post_replies.js rename to src/upgrades/1.3.0/sorted_sets_for_post_replies.js diff --git a/src/upgrades/global_and_user_language_keys.js b/src/upgrades/1.4.0/global_and_user_language_keys.js similarity index 100% rename from src/upgrades/global_and_user_language_keys.js rename to src/upgrades/1.4.0/global_and_user_language_keys.js diff --git a/src/upgrades/sorted_set_for_pinned_topics.js b/src/upgrades/1.4.0/sorted_set_for_pinned_topics.js similarity index 100% rename from src/upgrades/sorted_set_for_pinned_topics.js rename to src/upgrades/1.4.0/sorted_set_for_pinned_topics.js diff --git a/src/upgrades/flags_refactor.js b/src/upgrades/develop/flags_refactor.js similarity index 100% rename from src/upgrades/flags_refactor.js rename to src/upgrades/develop/flags_refactor.js diff --git a/src/upgrades/post_votes_zset.js b/src/upgrades/develop/post_votes_zset.js similarity index 100% rename from src/upgrades/post_votes_zset.js rename to src/upgrades/develop/post_votes_zset.js diff --git a/src/upgrades/config_urls_update.js b/src/upgrades/master/config_urls_update.js similarity index 100% rename from src/upgrades/config_urls_update.js rename to src/upgrades/master/config_urls_update.js diff --git a/src/upgrades/sound_settings.js b/src/upgrades/master/sound_settings.js similarity index 100% rename from src/upgrades/sound_settings.js rename to src/upgrades/master/sound_settings.js From e30917598983f595a8e1bc75e45e75ae112ecbe6 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 10 Mar 2017 16:37:16 -0500 Subject: [PATCH 14/16] eslinted template --- src/upgrades/TEMPLATE | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/upgrades/TEMPLATE b/src/upgrades/TEMPLATE index 5b79aa6148..7d451ae761 100644 --- a/src/upgrades/TEMPLATE +++ b/src/upgrades/TEMPLATE @@ -1,4 +1,5 @@ /* jslint node: true */ + 'use strict'; var db = require('../database'); @@ -11,5 +12,5 @@ module.exports = { timestamp: Date.UTC(2017, 0, 1), method: function (callback) { // Do stuff here... - } -}; \ No newline at end of file + }, +}; From f0059ec723c0c642330af9186aa35faffb0564c4 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 10 Mar 2017 19:07:05 -0500 Subject: [PATCH 15/16] fixed paths in upgrade scripts --- src/upgrades/1.0.0/chat_room_hashes.js | 2 +- src/upgrades/1.0.0/chat_upgrade.js | 2 +- src/upgrades/1.0.0/global_moderators.js | 2 +- src/upgrades/1.0.0/social_post_sharing.js | 4 ++-- src/upgrades/1.0.0/theme_to_active_plugins.js | 2 +- src/upgrades/1.0.0/user_best_posts.js | 4 ++-- src/upgrades/1.0.0/users_notvalidated.js | 4 ++-- src/upgrades/1.1.0/assign_topic_read_privilege.js | 6 +++--- src/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js | 6 +++--- src/upgrades/1.1.0/group_title_update.js | 6 +++--- src/upgrades/1.1.0/separate_upvote_downvote.js | 6 +++--- src/upgrades/1.1.0/user_post_count_per_tid.js | 6 +++--- src/upgrades/1.1.1/remove_negative_best_posts.js | 4 ++-- src/upgrades/1.1.1/upload_privileges.js | 6 +++--- src/upgrades/1.2.0/category_recent_tids.js | 2 +- src/upgrades/1.2.0/edit_delete_deletetopic_privileges.js | 6 +++--- src/upgrades/1.3.0/favourites_to_bookmarks.js | 6 +++--- src/upgrades/1.3.0/sorted_sets_for_post_replies.js | 6 +++--- src/upgrades/1.4.0/global_and_user_language_keys.js | 8 ++++---- src/upgrades/1.4.0/sorted_set_for_pinned_topics.js | 6 +++--- src/upgrades/{master => 1.4.4}/config_urls_update.js | 2 +- src/upgrades/{master => 1.4.4}/sound_settings.js | 6 +++--- src/upgrades/{develop => 1.5.0}/flags_refactor.js | 8 ++++---- src/upgrades/{develop => 1.5.0}/post_votes_zset.js | 4 ++-- src/upgrades/TEMPLATE | 2 +- 25 files changed, 58 insertions(+), 58 deletions(-) rename src/upgrades/{master => 1.4.4}/config_urls_update.js (95%) rename src/upgrades/{master => 1.4.4}/sound_settings.js (93%) rename src/upgrades/{develop => 1.5.0}/flags_refactor.js (94%) rename src/upgrades/{develop => 1.5.0}/post_votes_zset.js (83%) diff --git a/src/upgrades/1.0.0/chat_room_hashes.js b/src/upgrades/1.0.0/chat_room_hashes.js index bccf85282c..054b6c8d7e 100644 --- a/src/upgrades/1.0.0/chat_room_hashes.js +++ b/src/upgrades/1.0.0/chat_room_hashes.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); diff --git a/src/upgrades/1.0.0/chat_upgrade.js b/src/upgrades/1.0.0/chat_upgrade.js index ce49b501de..ae93ca95a5 100644 --- a/src/upgrades/1.0.0/chat_upgrade.js +++ b/src/upgrades/1.0.0/chat_upgrade.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); diff --git a/src/upgrades/1.0.0/global_moderators.js b/src/upgrades/1.0.0/global_moderators.js index bfe909a11b..3ba5f49955 100644 --- a/src/upgrades/1.0.0/global_moderators.js +++ b/src/upgrades/1.0.0/global_moderators.js @@ -8,7 +8,7 @@ module.exports = { name: 'Creating Global moderators group', timestamp: Date.UTC(2016, 0, 23), method: function (callback) { - var groups = require('../groups'); + var groups = require('../../groups'); async.waterfall([ function (next) { groups.exists('Global Moderators', next); diff --git a/src/upgrades/1.0.0/social_post_sharing.js b/src/upgrades/1.0.0/social_post_sharing.js index 22c4a8c7cf..03d303fe07 100644 --- a/src/upgrades/1.0.0/social_post_sharing.js +++ b/src/upgrades/1.0.0/social_post_sharing.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); @@ -10,7 +10,7 @@ module.exports = { name: 'Social: Post Sharing', timestamp: Date.UTC(2016, 1, 25), method: function (callback) { - var social = require('../social'); + var social = require('../../social'); async.parallel([ function (next) { social.setActivePostSharingNetworks(['facebook', 'google', 'twitter'], next); diff --git a/src/upgrades/1.0.0/theme_to_active_plugins.js b/src/upgrades/1.0.0/theme_to_active_plugins.js index 3adbca16dc..3a3bdd25eb 100644 --- a/src/upgrades/1.0.0/theme_to_active_plugins.js +++ b/src/upgrades/1.0.0/theme_to_active_plugins.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); diff --git a/src/upgrades/1.0.0/user_best_posts.js b/src/upgrades/1.0.0/user_best_posts.js index 126539c2b7..a832a71254 100644 --- a/src/upgrades/1.0.0/user_best_posts.js +++ b/src/upgrades/1.0.0/user_best_posts.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,7 +11,7 @@ module.exports = { name: 'Creating user best post sorted sets', timestamp: Date.UTC(2016, 0, 14), method: function (callback) { - var batch = require('../batch'); + var batch = require('../../batch'); batch.processSortedSet('posts:pid', function (ids, next) { async.eachSeries(ids, function (id, next) { diff --git a/src/upgrades/1.0.0/users_notvalidated.js b/src/upgrades/1.0.0/users_notvalidated.js index 9c23b6a539..5b5e60270b 100644 --- a/src/upgrades/1.0.0/users_notvalidated.js +++ b/src/upgrades/1.0.0/users_notvalidated.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,7 +11,7 @@ module.exports = { name: 'Creating users:notvalidated', timestamp: Date.UTC(2016, 0, 20), method: function (callback) { - var batch = require('../batch'); + var batch = require('../../batch'); var now = Date.now(); batch.processSortedSet('users:joindate', function (ids, next) { async.eachSeries(ids, function (id, next) { diff --git a/src/upgrades/1.1.0/assign_topic_read_privilege.js b/src/upgrades/1.1.0/assign_topic_read_privilege.js index 1e65d0d6f7..9f23cd7384 100644 --- a/src/upgrades/1.1.0/assign_topic_read_privilege.js +++ b/src/upgrades/1.1.0/assign_topic_read_privilege.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,8 +11,8 @@ module.exports = { name: 'Giving topics:read privs to any group that was previously allowed to Find & Access Category', timestamp: Date.UTC(2016, 4, 28), method: function (callback) { - var groupsAPI = require('../groups'); - var privilegesAPI = require('../privileges'); + var groupsAPI = require('../../groups'); + var privilegesAPI = require('../../privileges'); db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) { if (err) { diff --git a/src/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js b/src/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js index cf4af73436..85d14f6bbc 100644 --- a/src/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js +++ b/src/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,8 +11,8 @@ module.exports = { name: 'Dismiss flags from deleted topics', timestamp: Date.UTC(2016, 3, 29), method: function (callback) { - var posts = require('../posts'); - var topics = require('../topics'); + var posts = require('../../posts'); + var topics = require('../../topics'); var pids; var tids; diff --git a/src/upgrades/1.1.0/group_title_update.js b/src/upgrades/1.1.0/group_title_update.js index 5f82e97bfe..1c6065a3d4 100644 --- a/src/upgrades/1.1.0/group_title_update.js +++ b/src/upgrades/1.1.0/group_title_update.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,8 +11,8 @@ module.exports = { name: 'Group title from settings to user profile', timestamp: Date.UTC(2016, 3, 14), method: function (callback) { - var user = require('../user'); - var batch = require('../batch'); + var user = require('../../user'); + var batch = require('../../batch'); var count = 0; batch.processSortedSet('users:joindate', function (uids, next) { winston.verbose('upgraded ' + count + ' users'); diff --git a/src/upgrades/1.1.0/separate_upvote_downvote.js b/src/upgrades/1.1.0/separate_upvote_downvote.js index b313c7439d..c397a36a7b 100644 --- a/src/upgrades/1.1.0/separate_upvote_downvote.js +++ b/src/upgrades/1.1.0/separate_upvote_downvote.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,8 +11,8 @@ module.exports = { name: 'Store upvotes/downvotes separately', timestamp: Date.UTC(2016, 5, 13), method: function (callback) { - var batch = require('../batch'); - var posts = require('../posts'); + var batch = require('../../batch'); + var posts = require('../../posts'); var count = 0; batch.processSortedSet('posts:pid', function (pids, next) { winston.verbose('upgraded ' + count + ' posts'); diff --git a/src/upgrades/1.1.0/user_post_count_per_tid.js b/src/upgrades/1.1.0/user_post_count_per_tid.js index c0e86c4f52..7b27606c99 100644 --- a/src/upgrades/1.1.0/user_post_count_per_tid.js +++ b/src/upgrades/1.1.0/user_post_count_per_tid.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,8 +11,8 @@ module.exports = { name: 'Users post count per tid', timestamp: Date.UTC(2016, 3, 19), method: function (callback) { - var batch = require('../batch'); - var topics = require('../topics'); + var batch = require('../../batch'); + var topics = require('../../topics'); var count = 0; batch.processSortedSet('topics:tid', function (tids, next) { winston.verbose('upgraded ' + count + ' topics'); diff --git a/src/upgrades/1.1.1/remove_negative_best_posts.js b/src/upgrades/1.1.1/remove_negative_best_posts.js index 604d1d8234..22b6bb9ad1 100644 --- a/src/upgrades/1.1.1/remove_negative_best_posts.js +++ b/src/upgrades/1.1.1/remove_negative_best_posts.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,7 +11,7 @@ module.exports = { name: 'Removing best posts with negative scores', timestamp: Date.UTC(2016, 7, 5), method: function (callback) { - var batch = require('../batch'); + var batch = require('../../batch'); batch.processSortedSet('users:joindate', function (ids, next) { async.each(ids, function (id, next) { winston.verbose('processing uid ' + id); diff --git a/src/upgrades/1.1.1/upload_privileges.js b/src/upgrades/1.1.1/upload_privileges.js index b3268a2022..d84516efb7 100644 --- a/src/upgrades/1.1.1/upload_privileges.js +++ b/src/upgrades/1.1.1/upload_privileges.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); @@ -10,8 +10,8 @@ module.exports = { name: 'Giving upload privileges', timestamp: Date.UTC(2016, 6, 12), method: function (callback) { - var privilegesAPI = require('../privileges'); - var meta = require('../meta'); + var privilegesAPI = require('../../privileges'); + var meta = require('../../meta'); db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) { if (err) { diff --git a/src/upgrades/1.2.0/category_recent_tids.js b/src/upgrades/1.2.0/category_recent_tids.js index 18d27266f6..9256cde5f5 100644 --- a/src/upgrades/1.2.0/category_recent_tids.js +++ b/src/upgrades/1.2.0/category_recent_tids.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); diff --git a/src/upgrades/1.2.0/edit_delete_deletetopic_privileges.js b/src/upgrades/1.2.0/edit_delete_deletetopic_privileges.js index 5ba6f604c5..acc87b7b19 100644 --- a/src/upgrades/1.2.0/edit_delete_deletetopic_privileges.js +++ b/src/upgrades/1.2.0/edit_delete_deletetopic_privileges.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,8 +11,8 @@ module.exports = { name: 'Granting edit/delete/delete topic on existing categories', timestamp: Date.UTC(2016, 7, 7), method: function (callback) { - var groupsAPI = require('../groups'); - var privilegesAPI = require('../privileges'); + var groupsAPI = require('../../groups'); + var privilegesAPI = require('../../privileges'); db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) { if (err) { diff --git a/src/upgrades/1.3.0/favourites_to_bookmarks.js b/src/upgrades/1.3.0/favourites_to_bookmarks.js index 05b1e5b5f9..0888e69653 100644 --- a/src/upgrades/1.3.0/favourites_to_bookmarks.js +++ b/src/upgrades/1.3.0/favourites_to_bookmarks.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); @@ -11,7 +11,7 @@ module.exports = { timestamp: Date.UTC(2016, 9, 8), method: function (callback) { function upgradePosts(next) { - var batch = require('../batch'); + var batch = require('../../batch'); batch.processSortedSet('posts:pid', function (ids, next) { async.each(ids, function (id, next) { @@ -38,7 +38,7 @@ module.exports = { } function upgradeUsers(next) { - var batch = require('../batch'); + var batch = require('../../batch'); batch.processSortedSet('users:joindate', function (ids, next) { async.each(ids, function (id, next) { diff --git a/src/upgrades/1.3.0/sorted_sets_for_post_replies.js b/src/upgrades/1.3.0/sorted_sets_for_post_replies.js index 78fc96c7a1..b3ecd25f0a 100644 --- a/src/upgrades/1.3.0/sorted_sets_for_post_replies.js +++ b/src/upgrades/1.3.0/sorted_sets_for_post_replies.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,8 +11,8 @@ module.exports = { name: 'Sorted sets for post replies', timestamp: Date.UTC(2016, 9, 14), method: function (callback) { - var posts = require('../posts'); - var batch = require('../batch'); + var posts = require('../../posts'); + var batch = require('../../batch'); batch.processSortedSet('posts:pid', function (ids, next) { posts.getPostsFields(ids, ['pid', 'toPid', 'timestamp'], function (err, data) { if (err) { diff --git a/src/upgrades/1.4.0/global_and_user_language_keys.js b/src/upgrades/1.4.0/global_and_user_language_keys.js index 14f71c1fce..20548e6820 100644 --- a/src/upgrades/1.4.0/global_and_user_language_keys.js +++ b/src/upgrades/1.4.0/global_and_user_language_keys.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); @@ -10,9 +10,9 @@ module.exports = { name: 'Update global and user language keys', timestamp: Date.UTC(2016, 10, 22), method: function (callback) { - var user = require('../user'); - var meta = require('../meta'); - var batch = require('../batch'); + var user = require('../../user'); + var meta = require('../../meta'); + var batch = require('../../batch'); var newLanguage; async.parallel([ function (next) { diff --git a/src/upgrades/1.4.0/sorted_set_for_pinned_topics.js b/src/upgrades/1.4.0/sorted_set_for_pinned_topics.js index faa12f3cc8..a82ef80884 100644 --- a/src/upgrades/1.4.0/sorted_set_for_pinned_topics.js +++ b/src/upgrades/1.4.0/sorted_set_for_pinned_topics.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); @@ -11,8 +11,8 @@ module.exports = { name: 'Sorted set for pinned topics', timestamp: Date.UTC(2016, 10, 25), method: function (callback) { - var topics = require('../topics'); - var batch = require('../batch'); + var topics = require('../../topics'); + var batch = require('../../batch'); batch.processSortedSet('topics:tid', function (ids, next) { topics.getTopicsFields(ids, ['tid', 'cid', 'pinned', 'lastposttime'], function (err, data) { if (err) { diff --git a/src/upgrades/master/config_urls_update.js b/src/upgrades/1.4.4/config_urls_update.js similarity index 95% rename from src/upgrades/master/config_urls_update.js rename to src/upgrades/1.4.4/config_urls_update.js index bd4dfc5d83..4a9d1d703c 100644 --- a/src/upgrades/master/config_urls_update.js +++ b/src/upgrades/1.4.4/config_urls_update.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); diff --git a/src/upgrades/master/sound_settings.js b/src/upgrades/1.4.4/sound_settings.js similarity index 93% rename from src/upgrades/master/sound_settings.js rename to src/upgrades/1.4.4/sound_settings.js index 610b51f043..73b15e881a 100644 --- a/src/upgrades/master/sound_settings.js +++ b/src/upgrades/1.4.4/sound_settings.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); @@ -10,8 +10,8 @@ module.exports = { name: 'Update global and user sound settings', timestamp: Date.UTC(2017, 1, 25), method: function (callback) { - var meta = require('../meta'); - var batch = require('../batch'); + var meta = require('../../meta'); + var batch = require('../../batch'); var map = { 'notification.mp3': 'Default | Deedle-dum', diff --git a/src/upgrades/develop/flags_refactor.js b/src/upgrades/1.5.0/flags_refactor.js similarity index 94% rename from src/upgrades/develop/flags_refactor.js rename to src/upgrades/1.5.0/flags_refactor.js index deb1befc6c..7aff6bcfe7 100644 --- a/src/upgrades/develop/flags_refactor.js +++ b/src/upgrades/1.5.0/flags_refactor.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); @@ -10,9 +10,9 @@ module.exports = { name: 'Migrating flags to new schema', timestamp: Date.UTC(2016, 11, 7), method: function (callback) { - var batch = require('../batch'); - var posts = require('../posts'); - var flags = require('../flags'); + var batch = require('../../batch'); + var posts = require('../../posts'); + var flags = require('../../flags'); batch.processSortedSet('posts:pid', function (ids, next) { posts.getPostsByPids(ids, 1, function (err, posts) { diff --git a/src/upgrades/develop/post_votes_zset.js b/src/upgrades/1.5.0/post_votes_zset.js similarity index 83% rename from src/upgrades/develop/post_votes_zset.js rename to src/upgrades/1.5.0/post_votes_zset.js index c1eefb6688..d989602e23 100644 --- a/src/upgrades/develop/post_votes_zset.js +++ b/src/upgrades/1.5.0/post_votes_zset.js @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); @@ -10,7 +10,7 @@ module.exports = { name: 'New sorted set posts:votes', timestamp: Date.UTC(2017, 1, 27), method: function (callback) { - require('../batch').processSortedSet('posts:pid', function (pids, next) { + require('../../batch').processSortedSet('posts:pid', function (pids, next) { async.each(pids, function (pid, next) { db.getObjectFields('post:' + pid, ['upvotes', 'downvotes'], function (err, postData) { if (err || !postData) { diff --git a/src/upgrades/TEMPLATE b/src/upgrades/TEMPLATE index 7d451ae761..83cd01586d 100644 --- a/src/upgrades/TEMPLATE +++ b/src/upgrades/TEMPLATE @@ -2,7 +2,7 @@ 'use strict'; -var db = require('../database'); +var db = require('../../database'); var async = require('async'); var winston = require('winston'); From 34d24c4fd49e821b60585cf8b01ae99fdc9f7ef9 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 10 Mar 2017 19:07:42 -0500 Subject: [PATCH 16/16] Refactored upgrade scripts once more (for great justice!) #5467 --- src/upgrade.js | 77 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/src/upgrade.js b/src/upgrade.js index 8978f5e60a..081ddc5757 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -4,6 +4,7 @@ var async = require('async'); var path = require('path'); +var semver = require('semver'); var db = require('./database'); var utils = require('../public/src/utils'); @@ -55,24 +56,47 @@ var Upgrade = { ], }; +Upgrade.getAll = function (callback) { + async.waterfall([ + async.apply(utils.walk, path.join(__dirname, './upgrades')), + function (files, next) { + // Sort the upgrade scripts based on version + var versionA; + var versionB; + setImmediate(next, null, files.filter(function (file) { + return path.basename(file) !== 'TEMPLATE'; + }).sort(function (a, b) { + versionA = path.dirname(a).split('/').pop(); + versionB = path.dirname(b).split('/').pop(); + + return semver.compare(versionA, versionB); + })); + }, + ], callback); +}; + Upgrade.check = function (callback) { // Throw 'schema-out-of-date' if not all upgrade scripts have run - var all = Upgrade.available.reduce(function (memo, current) { - memo = memo.concat(current.upgrades); - return memo; - }, []); - - db.getSortedSetRange('schemaLog', 0, -1, function (err, executed) { - if (err) { - return callback(err); - } + async.waterfall([ + async.apply(Upgrade.getAll), + function (files, next) { + db.getSortedSetRange('schemaLog', 0, -1, function (err, executed) { + if (err) { + return callback(err); + } - var remainder = all.filter(function (name) { - return executed.indexOf(name) === -1; - }); + var remainder = files.filter(function (name) { + return executed.indexOf(path.basename(name, '.js')) === -1; + }); - callback(remainder.length > 1 ? new Error('schema-out-of-date') : null); - }); + next(remainder.length > 1 ? new Error('schema-out-of-date') : null); + }); + }, + ], callback); + // var all = Upgrade.available.reduce(function (memo, current) { + // memo = memo.concat(current.upgrades); + // return memo; + // }, []); }; Upgrade.run = function (callback) { @@ -80,20 +104,22 @@ Upgrade.run = function (callback) { var queue = []; var skipped = 0; - // Retrieve list of upgrades that have already been run - db.getSortedSetRange('schemaLog', 0, -1, function (err, completed) { + async.parallel({ + // Retrieve list of upgrades that have already been run + completed: async.apply(db.getSortedSetRange, 'schemaLog', 0, -1), + // ... and those available to be run + available: Upgrade.getAll, + }, function (err, data) { if (err) { return callback(err); } - queue = Upgrade.available.reduce(function (memo, cur) { - cur.upgrades.forEach(function (filename) { - if (completed.indexOf(filename) === -1) { - memo.push(path.join(__dirname, './upgrades', filename)); - } else { - skipped += 1; - } - }); + queue = data.available.reduce(function (memo, cur) { + if (data.completed.indexOf(path.basename(cur, '.js')) === -1) { + memo.push(cur); + } else { + skipped += 1; + } return memo; }, queue); @@ -124,9 +150,6 @@ Upgrade.runSingle = function (query, callback) { Upgrade.process = function (files, skipCount, callback) { process.stdout.write('OK'.green + ' | '.reset + String(files.length).cyan + ' script(s) found'.cyan + (skipCount > 0 ? ', '.cyan + String(skipCount).cyan + ' skipped'.cyan : '') + '\n'.reset); - // Do I need to sort the files here? we'll see. - // sort(); - async.eachSeries(files, function (file, next) { var scriptExport = require(file); var date = new Date(scriptExport.timestamp);