From a6c3be0bb1cc3900d291f1b73e93a7f3c44e2144 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Thu, 22 Sep 2016 18:21:49 +0300 Subject: [PATCH] store category recent tids for faster retrieval --- src/categories/recentreplies.js | 77 ++++++++++++++++----------------- src/categories/topics.js | 3 ++ src/topics/create.js | 3 ++ src/upgrade.js | 41 +++++++++++++++++- 4 files changed, 83 insertions(+), 41 deletions(-) diff --git a/src/categories/recentreplies.js b/src/categories/recentreplies.js index d9393c7c08..e7a1d0ba4e 100644 --- a/src/categories/recentreplies.js +++ b/src/categories/recentreplies.js @@ -32,6 +32,39 @@ module.exports = function(Categories) { ], callback); }; + Categories.updateRecentTid = function(cid, tid, callback) { + async.parallel({ + count: function(next) { + db.sortedSetCard('cid:' + cid + ':recent_tids', next); + }, + numRecentReplies: function(next) { + db.getObjectField('category:' + cid, 'numRecentReplies', next); + } + }, function(err, results) { + if (err) { + return callback(err); + } + + if (results.count < results.numRecentReplies) { + return db.sortedSetAdd('cid:' + cid + ':recent_tids', Date.now(), tid, callback); + } + async.waterfall([ + function(next) { + db.getSortedSetRangeWithScores('cid:' + cid + ':recent_tids', 0, results.count - results.numRecentReplies, next); + }, + function(data, next) { + if (!data.length) { + return next(); + } + db.sortedSetsRemoveRangeByScore(['cid:' + cid + ':recent_tids'], '-inf', data[data.length - 1].score, next); + }, + function(next) { + db.sortedSetAdd('cid:' + cid + ':recent_tids', Date.now(), tid, next); + } + ], callback); + }); + }; + Categories.getRecentTopicReplies = function(categoryData, uid, callback) { if (!Array.isArray(categoryData) || !categoryData.length) { return callback(); @@ -39,7 +72,10 @@ module.exports = function(Categories) { async.waterfall([ function(next) { - async.map(categoryData, getRecentTopicTids, next); + var keys = categoryData.map(function(category) { + return 'cid:' + category.cid + ':recent_tids'; + }); + db.getSortedSetsMembers(keys, next); }, function(results, next) { var tids = _.flatten(results); @@ -62,45 +98,6 @@ module.exports = function(Categories) { ], callback); }; - function getRecentTopicTids(category, callback) { - var count = parseInt(category.numRecentReplies, 10); - if (!count) { - return callback(null, []); - } - - if (count === 1) { - async.waterfall([ - function (next) { - db.getSortedSetRevRange('cid:' + category.cid + ':pids', 0, 0, next); - }, - function (pid, next) { - posts.getPostField(pid, 'tid', next); - }, - function (tid, next) { - next(null, [tid]); - } - ], callback); - return; - } - - async.parallel({ - pinnedTids: function(next) { - db.getSortedSetRevRangeByScore('cid:' + category.cid + ':tids', 0, -1, '+inf', Date.now(), next); - }, - tids: function(next) { - db.getSortedSetRevRangeByScore('cid:' + category.cid + ':tids', 0, Math.max(1, count), Date.now(), '-inf', next); - } - }, function(err, results) { - if (err) { - return callback(err); - } - - results.tids = results.tids.concat(results.pinnedTids); - - callback(null, results.tids); - }); - } - function getTopics(tids, callback) { var topicData; async.waterfall([ diff --git a/src/categories/topics.js b/src/categories/topics.js index 05b20668cf..b1a0bfae54 100644 --- a/src/categories/topics.js +++ b/src/categories/topics.js @@ -89,6 +89,9 @@ module.exports = function(Categories) { db.sortedSetAdd('cid:' + cid + ':tids', postData.timestamp, postData.tid, next); } }, + function(next){ + Categories.updateRecentTid(cid, postData.tid, next); + }, function(next) { db.sortedSetIncrBy('cid:' + cid + ':tids:posts', 1, postData.tid, next); } diff --git a/src/topics/create.js b/src/topics/create.js index db4976678c..85491a0643 100644 --- a/src/topics/create.js +++ b/src/topics/create.js @@ -63,6 +63,9 @@ module.exports = function(Topics) { 'cid:' + topicData.cid + ':uid:' + topicData.uid + ':tids' ], timestamp, topicData.tid, next); }, + function(next) { + categories.updateRecentTid(topicData.cid, topicData.tid, next); + }, function(next) { user.addTopicIdToUser(topicData.uid, topicData.tid, timestamp, next); }, diff --git a/src/upgrade.js b/src/upgrade.js index 317f1837da..c6fbe767e3 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -10,7 +10,7 @@ var db = require('./database'), schemaDate, thisSchemaDate, // IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema - latestSchema = Date.UTC(2016, 8, 7); + latestSchema = Date.UTC(2016, 8, 22); Upgrade.check = function(callback) { db.get('schemaDate', function(err, value) { @@ -811,6 +811,45 @@ Upgrade.upgrade = function(callback) { winston.info('[2016/08/07] Granting edit/delete/delete topic on existing categories - skipped!'); next(); } + }, + function(next) { + thisSchemaDate = Date.UTC(2016, 8, 22); + + if (schemaDate < thisSchemaDate) { + updatesMade = true; + winston.info('[2016/09/22] Setting category recent tids'); + + + db.getSortedSetRange('categories:cid', 0, -1, function(err, cids) { + if (err) { + return next(err); + } + + async.eachSeries(cids, function(cid, next) { + db.getSortedSetRevRange('cid:' + cid + ':pids', 0, 0, function(err, pid) { + if (err || !pid) { + return next(err); + } + db.getObjectFields('post:' + pid, ['tid', 'timestamp'], function(err, postData) { + if (err || !postData || !postData.tid) { + return next(err); + } + db.sortedSetAdd('cid:' + cid + ':recent_tids', postData.timestamp, postData.tid, next); + }); + }); + }, function(err) { + if (err) { + return next(err); + } + + winston.info('[2016/09/22] Setting category recent tids - done'); + Upgrade.update(thisSchemaDate, next); + }); + }); + } else { + winston.info('[2016/09/22] Setting category recent tids - skipped!'); + next(); + } } // Add new schema updates here // IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema IN LINE 24!!!