From 42b5cdaae33cdfa2ddd21d19c75cf0db340a19e8 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 12 Apr 2017 17:30:20 -0400 Subject: [PATCH 1/5] progress bars! omg --- src/upgrade.js | 31 +++++++++++++- .../1.5.0/moderation_history_refactor.js | 42 ++++++++++++------- src/upgrades/1.5.0/post_votes_zset.js | 35 ++++++++++------ 3 files changed, 80 insertions(+), 28 deletions(-) diff --git a/src/upgrade.js b/src/upgrade.js index 60b074a953..06e2e7147e 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -157,6 +157,13 @@ Upgrade.process = function (files, skipCount, callback) { var scriptExport = require(file); var date = new Date(scriptExport.timestamp); var version = path.dirname(file).split('/').pop(); + var progress = { + current: 0, + total: 0, + incr: Upgrade.incrementProgress, + script: scriptExport, + date: date, + }; process.stdout.write(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '... '); @@ -168,7 +175,9 @@ Upgrade.process = function (files, skipCount, callback) { } // Do the upgrade... - scriptExport.method(function (err) { + scriptExport.method.bind({ + progress: progress, + })(function (err) { if (err) { process.stdout.write('error\n'.red); return next(err); @@ -177,6 +186,12 @@ Upgrade.process = function (files, skipCount, callback) { // Record success in schemaLog db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js')); + if (progress.total > 0) { + process.stdout.clearLine(); + process.stdout.cursorTo(0); + process.stdout.write(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '... '); + } + process.stdout.write('OK\n'.green); next(); }); @@ -192,4 +207,18 @@ Upgrade.process = function (files, skipCount, callback) { ], callback); }; +Upgrade.incrementProgress = function () { + this.current += 1; + + // Redraw the progress bar + var percentage = Math.floor((this.current / this.total) * 100) + '%'; + var filled = Math.floor((this.current / this.total) * 15); + var unfilled = 15 - filled; + process.stdout.clearLine(); + process.stdout.cursorTo(0); + + process.stdout.write(' → '.white + String('[' + [this.date.getUTCFullYear(), this.date.getUTCMonth() + 1, this.date.getUTCDate()].join('/') + '] ').gray + String(this.script.name).reset + '... '); + process.stdout.write('[' + (filled ? new Array(filled).join('#') : '') + new Array(unfilled).join(' ') + '] (' + this.current + '/' + this.total + ') ' + percentage); +}; + module.exports = Upgrade; diff --git a/src/upgrades/1.5.0/moderation_history_refactor.js b/src/upgrades/1.5.0/moderation_history_refactor.js index 73b4e8f6d7..b167246e5f 100644 --- a/src/upgrades/1.5.0/moderation_history_refactor.js +++ b/src/upgrades/1.5.0/moderation_history_refactor.js @@ -11,20 +11,32 @@ module.exports = { name: 'Update moderation notes to zset', timestamp: Date.UTC(2017, 2, 22), method: function (callback) { - batch.processSortedSet('users:joindate', function (ids, next) { - async.each(ids, function (uid, next) { - db.getObjectField('user:' + uid, 'moderationNote', function (err, moderationNote) { - if (err || !moderationNote) { - return next(err); - } - var note = { - uid: 1, - note: moderationNote, - timestamp: Date.now(), - }; - db.sortedSetAdd('uid:' + uid + ':moderation:notes', note.timestamp, JSON.stringify(note), next); - }); - }, next); - }, callback); + var progress = this.progress; + + db.sortedSetCard('users:joindate', function (err, numPosts) { + if (err) { + return callback(err); + } + + progress.total = numPosts; + + batch.processSortedSet('users:joindate', function (ids, next) { + async.each(ids, function (uid, next) { + db.getObjectField('user:' + uid, 'moderationNote', function (err, moderationNote) { + if (err || !moderationNote) { + return next(err); + } + var note = { + uid: 1, + note: moderationNote, + timestamp: Date.now(), + }; + + progress.incr(); + db.sortedSetAdd('uid:' + uid + ':moderation:notes', note.timestamp, JSON.stringify(note), next); + }); + }, next); + }, callback); + }); }, }; diff --git a/src/upgrades/1.5.0/post_votes_zset.js b/src/upgrades/1.5.0/post_votes_zset.js index d989602e23..df5a4c522f 100644 --- a/src/upgrades/1.5.0/post_votes_zset.js +++ b/src/upgrades/1.5.0/post_votes_zset.js @@ -10,17 +10,28 @@ module.exports = { name: 'New sorted set posts:votes', timestamp: Date.UTC(2017, 1, 27), method: function (callback) { - require('../../batch').processSortedSet('posts:pid', function (pids, next) { - async.each(pids, function (pid, next) { - db.getObjectFields('post:' + pid, ['upvotes', 'downvotes'], function (err, postData) { - if (err || !postData) { - return next(err); - } - - var votes = parseInt(postData.upvotes || 0, 10) - parseInt(postData.downvotes || 0, 10); - db.sortedSetAdd('posts:votes', votes, pid, next); - }); - }, next); - }, {}, callback); + var progress = this.progress; + + db.sortedSetCard('posts:pid', function (err, numPosts) { + if (err) { + return callback(err); + } + + progress.total = numPosts; + + require('../../batch').processSortedSet('posts:pid', function (pids, next) { + async.eachSeries(pids, function (pid, next) { + db.getObjectFields('post:' + pid, ['upvotes', 'downvotes'], function (err, postData) { + if (err || !postData) { + return next(err); + } + + var votes = parseInt(postData.upvotes || 0, 10) - parseInt(postData.downvotes || 0, 10); + progress.incr(); + db.sortedSetAdd('posts:votes', votes, pid, next); + }); + }, next); + }, {}, callback); + }); }, }; From 33082d90cc53b95aaf45d653967f497bc3f3b787 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sun, 16 Apr 2017 13:25:01 -0400 Subject: [PATCH 2/5] updated logic to make it a bit simpler to implement per upgrade script --- src/batch.js | 12 +++++ src/upgrade.js | 19 +++++--- .../1.5.0/moderation_history_refactor.js | 44 ++++++++----------- src/upgrades/1.5.0/post_votes_zset.js | 36 +++++++-------- 4 files changed, 58 insertions(+), 53 deletions(-) diff --git a/src/batch.js b/src/batch.js index 475c4aa7ca..f15d06a453 100644 --- a/src/batch.js +++ b/src/batch.js @@ -21,6 +21,17 @@ exports.processSortedSet = function (setKey, process, options, callback) { return callback(new Error('[[error:process-not-a-function]]')); } + // Progress bar handling (upgrade scripts) + if (options.progress) { + db.sortedSetCard(setKey, function (err, total) { + if (err) { + // Unable to get total, do nothing. + } else { + options.progress.total = total; + } + }); + } + // use the fast path if possible if (db.processSortedSet && typeof options.doneIf !== 'function' && !utils.isNumber(options.alwaysStartAt)) { return db.processSortedSet(setKey, process, options.batch || DEFAULT_BATCH_SIZE, callback); @@ -53,6 +64,7 @@ exports.processSortedSet = function (setKey, process, options, callback) { } start += utils.isNumber(options.alwaysStartAt) ? options.alwaysStartAt : batch + 1; stop = start + batch; + next(); }); }); diff --git a/src/upgrade.js b/src/upgrade.js index 9b2bf8b29e..5352a091f9 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -7,7 +7,7 @@ var path = require('path'); var semver = require('semver'); var db = require('./database'); -var utils = require('../public/src/utils'); +var file = require('../src/file'); /* * Need to write an upgrade script for NodeBB? Cool. @@ -58,7 +58,7 @@ var Upgrade = { Upgrade.getAll = function (callback) { async.waterfall([ - async.apply(utils.walk, path.join(__dirname, './upgrades')), + async.apply(file.walk, path.join(__dirname, './upgrades')), function (files, next) { // Sort the upgrade scripts based on version var versionA; @@ -128,7 +128,7 @@ Upgrade.runSingle = function (query, callback) { process.stdout.write('\nParsing upgrade scripts... '); async.waterfall([ - async.apply(utils.walk, path.join(__dirname, './upgrades')), + async.apply(file.walk, path.join(__dirname, './upgrades')), function (files, next) { next(null, files.filter(function (file) { return path.basename(file, '.js') === query; @@ -207,14 +207,19 @@ Upgrade.incrementProgress = function () { this.current += 1; // Redraw the progress bar - var percentage = Math.floor((this.current / this.total) * 100) + '%'; - var filled = Math.floor((this.current / this.total) * 15); - var unfilled = 15 - filled; + var percentage = 0; + var filled = 0; + var unfilled = 15; + if (this.total) { + percentage = Math.floor((this.current / this.total) * 100) + '%'; + filled = Math.floor((this.current / this.total) * 15); + unfilled = 15 - filled; + } process.stdout.clearLine(); process.stdout.cursorTo(0); process.stdout.write(' → '.white + String('[' + [this.date.getUTCFullYear(), this.date.getUTCMonth() + 1, this.date.getUTCDate()].join('/') + '] ').gray + String(this.script.name).reset + '... '); - process.stdout.write('[' + (filled ? new Array(filled).join('#') : '') + new Array(unfilled).join(' ') + '] (' + this.current + '/' + this.total + ') ' + percentage); + process.stdout.write('[' + (filled ? new Array(filled).join('#') : '') + new Array(unfilled).join(' ') + '] (' + this.current + '/' + (this.total || '??') + ') ' + percentage); }; module.exports = Upgrade; diff --git a/src/upgrades/1.5.0/moderation_history_refactor.js b/src/upgrades/1.5.0/moderation_history_refactor.js index b167246e5f..f98e1c62db 100644 --- a/src/upgrades/1.5.0/moderation_history_refactor.js +++ b/src/upgrades/1.5.0/moderation_history_refactor.js @@ -13,30 +13,24 @@ module.exports = { method: function (callback) { var progress = this.progress; - db.sortedSetCard('users:joindate', function (err, numPosts) { - if (err) { - return callback(err); - } - - progress.total = numPosts; - - batch.processSortedSet('users:joindate', function (ids, next) { - async.each(ids, function (uid, next) { - db.getObjectField('user:' + uid, 'moderationNote', function (err, moderationNote) { - if (err || !moderationNote) { - return next(err); - } - var note = { - uid: 1, - note: moderationNote, - timestamp: Date.now(), - }; - - progress.incr(); - db.sortedSetAdd('uid:' + uid + ':moderation:notes', note.timestamp, JSON.stringify(note), next); - }); - }, next); - }, callback); - }); + batch.processSortedSet('users:joindate', function (ids, next) { + async.each(ids, function (uid, next) { + db.getObjectField('user:' + uid, 'moderationNote', function (err, moderationNote) { + if (err || !moderationNote) { + return next(err); + } + var note = { + uid: 1, + note: moderationNote, + timestamp: Date.now(), + }; + + progress.incr(); + db.sortedSetAdd('uid:' + uid + ':moderation:notes', note.timestamp, JSON.stringify(note), next); + }); + }, next); + }, { + progress: this.progress, + }, callback); }, }; diff --git a/src/upgrades/1.5.0/post_votes_zset.js b/src/upgrades/1.5.0/post_votes_zset.js index df5a4c522f..b1d5b0cae4 100644 --- a/src/upgrades/1.5.0/post_votes_zset.js +++ b/src/upgrades/1.5.0/post_votes_zset.js @@ -12,26 +12,20 @@ module.exports = { method: function (callback) { var progress = this.progress; - db.sortedSetCard('posts:pid', function (err, numPosts) { - if (err) { - return callback(err); - } - - progress.total = numPosts; - - require('../../batch').processSortedSet('posts:pid', function (pids, next) { - async.eachSeries(pids, function (pid, next) { - db.getObjectFields('post:' + pid, ['upvotes', 'downvotes'], function (err, postData) { - if (err || !postData) { - return next(err); - } - - var votes = parseInt(postData.upvotes || 0, 10) - parseInt(postData.downvotes || 0, 10); - progress.incr(); - db.sortedSetAdd('posts:votes', votes, pid, next); - }); - }, next); - }, {}, callback); - }); + require('../../batch').processSortedSet('posts:pid', function (pids, next) { + async.eachSeries(pids, function (pid, next) { + db.getObjectFields('post:' + pid, ['upvotes', 'downvotes'], function (err, postData) { + if (err || !postData) { + return next(err); + } + + progress.incr(); + var votes = parseInt(postData.upvotes || 0, 10) - parseInt(postData.downvotes || 0, 10); + db.sortedSetAdd('posts:votes', votes, pid, next); + }); + }, next); + }, { + progress: this.progress, + }, callback); }, }; From 8fc61232305a6c702a5d1325836000de2883cf3b Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sun, 16 Apr 2017 13:29:36 -0400 Subject: [PATCH 3/5] fixing accidental eachSeries --- src/upgrades/1.5.0/post_votes_zset.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/upgrades/1.5.0/post_votes_zset.js b/src/upgrades/1.5.0/post_votes_zset.js index b1d5b0cae4..7feabe8169 100644 --- a/src/upgrades/1.5.0/post_votes_zset.js +++ b/src/upgrades/1.5.0/post_votes_zset.js @@ -13,7 +13,7 @@ module.exports = { var progress = this.progress; require('../../batch').processSortedSet('posts:pid', function (pids, next) { - async.eachSeries(pids, function (pid, next) { + async.each(pids, function (pid, next) { db.getObjectFields('post:' + pid, ['upvotes', 'downvotes'], function (err, postData) { if (err || !postData) { return next(err); From 3022a1920bb842d327c5c2796949f3884f502a72 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sun, 16 Apr 2017 13:37:17 -0400 Subject: [PATCH 4/5] added progress bars to all upgrade scripts processing posts:pid sorted set --- src/upgrades/1.0.0/user_best_posts.js | 6 +++++- src/upgrades/1.1.0/separate_upvote_downvote.js | 8 +++++++- src/upgrades/1.3.0/favourites_to_bookmarks.js | 7 ++++++- src/upgrades/1.3.0/sorted_sets_for_post_replies.js | 6 ++++++ src/upgrades/1.5.0/flags_refactor.js | 5 +++++ 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/upgrades/1.0.0/user_best_posts.js b/src/upgrades/1.0.0/user_best_posts.js index a832a71254..560ef7b29c 100644 --- a/src/upgrades/1.0.0/user_best_posts.js +++ b/src/upgrades/1.0.0/user_best_posts.js @@ -12,6 +12,7 @@ module.exports = { timestamp: Date.UTC(2016, 0, 14), method: function (callback) { var batch = require('../../batch'); + var progress = this.progress; batch.processSortedSet('posts:pid', function (ids, next) { async.eachSeries(ids, function (id, next) { @@ -24,8 +25,11 @@ module.exports = { } winston.verbose('processing pid: ' + postData.pid + ' uid: ' + postData.uid + ' votes: ' + postData.votes); db.sortedSetAdd('uid:' + postData.uid + ':posts:votes', postData.votes, postData.pid, next); + progress.incr(); }); }, next); - }, {}, callback); + }, { + progress: progress, + }, callback); }, }; diff --git a/src/upgrades/1.1.0/separate_upvote_downvote.js b/src/upgrades/1.1.0/separate_upvote_downvote.js index c397a36a7b..3f78667dc5 100644 --- a/src/upgrades/1.1.0/separate_upvote_downvote.js +++ b/src/upgrades/1.1.0/separate_upvote_downvote.js @@ -14,6 +14,8 @@ module.exports = { var batch = require('../../batch'); var posts = require('../../posts'); var count = 0; + var progress = this.progress; + batch.processSortedSet('posts:pid', function (pids, next) { winston.verbose('upgraded ' + count + ' posts'); count += pids.length; @@ -43,8 +45,12 @@ module.exports = { } else { next(); } + + progress.incr(); }, next); }, next); - }, {}, callback); + }, { + progress: progress, + }, callback); }, }; diff --git a/src/upgrades/1.3.0/favourites_to_bookmarks.js b/src/upgrades/1.3.0/favourites_to_bookmarks.js index 0888e69653..bbdc09ec4c 100644 --- a/src/upgrades/1.3.0/favourites_to_bookmarks.js +++ b/src/upgrades/1.3.0/favourites_to_bookmarks.js @@ -12,9 +12,12 @@ module.exports = { method: function (callback) { function upgradePosts(next) { var batch = require('../../batch'); + var progress = this.progress; batch.processSortedSet('posts:pid', function (ids, next) { async.each(ids, function (id, next) { + progress.incr(); + async.waterfall([ function (next) { db.rename('pid:' + id + ':users_favourited', 'pid:' + id + ':users_bookmarked', next); @@ -34,7 +37,9 @@ module.exports = { }, ], next); }, next); - }, {}, next); + }, { + progress: progress, + }, next); } function upgradeUsers(next) { diff --git a/src/upgrades/1.3.0/sorted_sets_for_post_replies.js b/src/upgrades/1.3.0/sorted_sets_for_post_replies.js index b3ecd25f0a..1cce287829 100644 --- a/src/upgrades/1.3.0/sorted_sets_for_post_replies.js +++ b/src/upgrades/1.3.0/sorted_sets_for_post_replies.js @@ -13,12 +13,16 @@ module.exports = { method: function (callback) { var posts = require('../../posts'); var batch = require('../../batch'); + var progress = this.progress; + batch.processSortedSet('posts:pid', function (ids, next) { posts.getPostsFields(ids, ['pid', 'toPid', 'timestamp'], function (err, data) { if (err) { return next(err); } + progress.incr(); + async.eachSeries(data, function (postData, next) { if (!parseInt(postData.toPid, 10)) { return next(null); @@ -30,6 +34,8 @@ module.exports = { ], next); }, next); }); + }, { + progress: progress, }, callback); }, }; diff --git a/src/upgrades/1.5.0/flags_refactor.js b/src/upgrades/1.5.0/flags_refactor.js index 7aff6bcfe7..08fc1831d8 100644 --- a/src/upgrades/1.5.0/flags_refactor.js +++ b/src/upgrades/1.5.0/flags_refactor.js @@ -13,6 +13,7 @@ module.exports = { var batch = require('../../batch'); var posts = require('../../posts'); var flags = require('../../flags'); + var progress = this.progress; batch.processSortedSet('posts:pid', function (ids, next) { posts.getPostsByPids(ids, 1, function (err, posts) { @@ -80,10 +81,14 @@ module.exports = { } else { next(err); } + + progress.incr(); }); }); }, next); }); + }, { + progress: this.progress, }, callback); }, }; From b25fbd1e2e134b8a6f99a030cda597532901667e Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sun, 16 Apr 2017 13:39:09 -0400 Subject: [PATCH 5/5] fix broken upgrade script --- src/upgrades/1.3.0/favourites_to_bookmarks.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/upgrades/1.3.0/favourites_to_bookmarks.js b/src/upgrades/1.3.0/favourites_to_bookmarks.js index bbdc09ec4c..065f02c94a 100644 --- a/src/upgrades/1.3.0/favourites_to_bookmarks.js +++ b/src/upgrades/1.3.0/favourites_to_bookmarks.js @@ -10,9 +10,10 @@ module.exports = { name: 'Favourites to Bookmarks', timestamp: Date.UTC(2016, 9, 8), method: function (callback) { + var progress = this.progress; + function upgradePosts(next) { var batch = require('../../batch'); - var progress = this.progress; batch.processSortedSet('posts:pid', function (ids, next) { async.each(ids, function (id, next) {