From 5fc760c6c4d4b9869b64d6e17360f74334d9c7e7 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sat, 27 Sep 2014 17:41:49 -0400 Subject: [PATCH 1/4] unread changes --- public/src/forum/infinitescroll.js | 6 +- src/database/mongo/sorted.js | 26 ++++++--- src/database/redis/sorted.js | 4 ++ src/topics/unread.js | 88 +++++++++++++++++------------- 4 files changed, 76 insertions(+), 48 deletions(-) diff --git a/public/src/forum/infinitescroll.js b/public/src/forum/infinitescroll.js index 1a8216397a..6f527f1fa3 100644 --- a/public/src/forum/infinitescroll.js +++ b/public/src/forum/infinitescroll.js @@ -15,9 +15,9 @@ define('forum/infinitescroll', function() { topOffset = _topOffest || 0; $(window).off('scroll', onScroll).on('scroll', onScroll); - if ($(document).height() === $(window).height()) { - callback(1); - } + // if ($(document).height() === $(window).height()) { + // callback(1); + // } }; function onScroll() { diff --git a/src/database/mongo/sorted.js b/src/database/mongo/sorted.js index 195de65adf..7c085cd48c 100644 --- a/src/database/mongo/sorted.js +++ b/src/database/mongo/sorted.js @@ -136,14 +136,18 @@ module.exports = function(db, module) { }; module.getSortedSetRangeByScore = function(key, start, count, min, max, callback) { - getSortedSetRangeByScore(key, start, count, min, max, 1, callback); + getSortedSetRangeByScore(key, start, count, min, max, 1, false, callback); }; module.getSortedSetRevRangeByScore = function(key, start, count, max, min, callback) { - getSortedSetRangeByScore(key, start, count, min, max, -1, callback); + getSortedSetRangeByScore(key, start, count, min, max, -1, false, callback); }; - function getSortedSetRangeByScore(key, start, count, min, max, sort, callback) { + module.getSortedSetRevRangeByScoreWithScores = function(key, start, count, max, min, callback) { + getSortedSetRangeByScore(key, start, count, min, max, -1, true, callback); + }; + + function getSortedSetRangeByScore(key, start, count, min, max, sort, withScores, callback) { if (!key) { return callback(); } @@ -159,7 +163,13 @@ module.exports = function(db, module) { scoreQuery['$lte'] = max; } - db.collection('objects').find({_key:key, score: scoreQuery}, {fields:{value:1}}) + var fields = {_id: 0}; + fields['value'] = 1; + if (withScores) { + fields['score'] = 1; + } + + db.collection('objects').find({_key:key, score: scoreQuery}, {fields: fields}) .limit(count) .skip(start) .sort({score: sort}) @@ -168,9 +178,11 @@ module.exports = function(db, module) { return callback(err); } - data = data.map(function(item) { - return item.value; - }); + if (!withScores) { + data = data.map(function(item) { + return item.value; + }); + } callback(err, data); }); diff --git a/src/database/redis/sorted.js b/src/database/redis/sorted.js index 185fb3d420..f9d939a4f7 100644 --- a/src/database/redis/sorted.js +++ b/src/database/redis/sorted.js @@ -98,6 +98,10 @@ module.exports = function(redisClient, module) { redisClient.zrevrangebyscore([key, max, min, 'LIMIT', start, count], callback); }; + module.getSortedSetRevRangeByScoreWithScores = function(key, start, count, max, min, callback) { + redisClient.zrevrangebyscore([key, max, min, 'WITHSCORES', 'LIMIT', start, count], callback); + }; + module.sortedSetCount = function(key, min, max, callback) { redisClient.zcount(key, min, max, callback); }; diff --git a/src/topics/unread.js b/src/topics/unread.js index 10dd9e53ab..f1233b8eac 100644 --- a/src/topics/unread.js +++ b/src/topics/unread.js @@ -69,12 +69,17 @@ module.exports = function(Topics) { return callback(null, []); } + var yesterday = Date.now() - 86400000; + async.parallel({ ignoredCids: function(next) { user.getIgnoredCategories(uid, next); }, recentTids: function(next) { - Topics.getLatestTids(0, -1, 'day', next); + db.getSortedSetRevRangeByScoreWithScores('topics:recent', 0, -1, Infinity, yesterday, next); + }, + userScores: function(next) { + db.getSortedSetRevRangeByScoreWithScores('uid:' + uid + ':tids_read', 0, -1, Infinity, yesterday, next); } }, function(err, results) { if (err) { @@ -85,28 +90,30 @@ module.exports = function(Topics) { return callback(null, []); } - Topics.hasReadTopics(results.recentTids, uid, function(err, read) { + var userRead = {}; + results.userScores.forEach(function(userItem) { + userRead[userItem.value] = userItem.score; + }); + + + var tids = results.recentTids.filter(function(recentTopic, index) { + return !userRead[recentTopic.value] || recentTopic.score > userRead[recentTopic.value]; + }).map(function(topic) { + return topic.value; + }); + + filterTopics(uid, tids, results.ignoredCids, function(err, tids) { if (err) { return callback(err); } - var tids = results.recentTids.filter(function(tid, index) { - return !read[index]; - }); - - filterTopics(uid, tids, results.ignoredCids, function(err, tids) { - if (err) { - return callback(err); - } - - if (stop === -1) { - tids = tids.slice(start); - } else { - tids = tids.slice(start, stop + 1); - } + if (stop === -1) { + tids = tids.slice(start); + } else { + tids = tids.slice(start, stop + 1); + } - callback(err, tids); - }); + callback(err, tids); }); }); }; @@ -167,12 +174,7 @@ module.exports = function(Topics) { }; Topics.markAsUnreadForAll = function(tid, callback) { - db.delete('tid:' + tid + ':read_by_uid', function(err) { - if(err) { - return callback(err); - } - Topics.markCategoryUnreadForAll(tid, callback); - }); + Topics.markCategoryUnreadForAll(tid, callback); }; Topics.markAsRead = function(tids, uid, callback) { @@ -181,13 +183,15 @@ module.exports = function(Topics) { return callback(); } tids = tids.filter(Boolean); - var keys = tids.map(function(tid) { - return 'tid:' + tid + ':read_by_uid'; + + var now = Date.now(); + var scores = tids.map(function(tid) { + return now; }); async.parallel({ markRead: function(next) { - db.setsAdd(keys, uid, next); + db.sortedSetAdd('uid:' + uid + ':tids_read', scores, tids, next); }, topicData: function(next) { Topics.getTopicsFields(tids, ['cid'], next); @@ -238,21 +242,29 @@ module.exports = function(Topics) { })); } - var sets = []; - - for (var i = 0, ii = tids.length; i < ii; i++) { - sets.push('tid:' + tids[i] + ':read_by_uid'); - } + async.parallel({ + recentScores: function(next) { + db.sortedSetScores('topics:recent', tids, next); + }, + userScores: function(next) { + db.sortedSetScores('uid:' + uid + ':tids_read', tids, next); + } + }, function(err, results) { + if (err) { + return callback(err); + } + var result = tids.map(function(tid, index) { + return !!(results.userScores[index] && results.userScores[index] >= results.recentScores[index]); + }); - db.isMemberOfSets(sets, uid, callback); + callback(null, result); + }); }; Topics.hasReadTopic = function(tid, uid, callback) { - if(!parseInt(uid, 10)) { - return callback(null, false); - } - - db.isSetMember('tid:' + tid + ':read_by_uid', uid, callback); + Topics.hasReadTopics([tid], uid, function(err, hasRead) { + callback(err, Array.isArray(hasRead) && hasRead.length ? hasRead[0] : false); + }); }; From 34106aa2096728858a1aa71924553a14e8a4cded Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sat, 27 Sep 2014 17:46:39 -0400 Subject: [PATCH 2/4] fix fields --- src/database/mongo/sorted.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/database/mongo/sorted.js b/src/database/mongo/sorted.js index 7c085cd48c..3714f5549d 100644 --- a/src/database/mongo/sorted.js +++ b/src/database/mongo/sorted.js @@ -104,7 +104,12 @@ module.exports = function(db, module) { if (!key) { return callback(); } - db.collection('objects').find({_key:key}, {fields: {_id: 0, value: 1, score: 1}}) + + var fields = {_id: 0, value: 1}; + if (withScores) { + fields['score'] = 1; + } + db.collection('objects').find({_key:key}, {fields: fields}) .limit(stop - start + 1) .skip(start) .sort({score: sort}) @@ -163,8 +168,7 @@ module.exports = function(db, module) { scoreQuery['$lte'] = max; } - var fields = {_id: 0}; - fields['value'] = 1; + var fields = {_id: 0, value: 1}; if (withScores) { fields['score'] = 1; } From 6535a730fd00fc25e63ee054e309dd10d4eaa957 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sat, 27 Sep 2014 17:57:56 -0400 Subject: [PATCH 3/4] upgrade script to delete tid::read_by_uid --- src/upgrade.js | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/upgrade.js b/src/upgrade.js index bc67ab7e2d..65bbdd7a2d 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -19,7 +19,7 @@ var db = require('./database'), schemaDate, thisSchemaDate, // IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema - latestSchema = Date.UTC(2014, 8, 8); + latestSchema = Date.UTC(2014, 8, 27); Upgrade.check = function(callback) { db.get('schemaDate', function(err, value) { @@ -1013,6 +1013,35 @@ Upgrade.upgrade = function(callback) { winston.info('[2014/9/8] Deleting old notifications skipped'); next(); } + }, + function(next) { + thisSchemaDate = Date.UTC(2014, 8, 27); + if (schemaDate < thisSchemaDate) { + winston.info('[2014/9/27] Deleting tid::read_by_uid...'); + + db.getSortedSetRange('topics:tid', 0, -1, function(err, tids) { + if (err) { + return next(err); + } + tids = tids.filter(Boolean); + var readKeys = tids.map(function(tid) { + return 'tid:' + tid + ':read_by_uid'; + }); + + db.deleteAll(readKeys, function(err, results) { + if (err) { + winston.error('[2014/9/27] Error encountered while deleting tid::read_by_uid'); + return next(err); + } + + winston.info('[2014/9/27] Deleted tid::read_by_uid'); + Upgrade.update(thisSchemaDate, next); + }); + }); + } else { + winston.info('[2014/9/27] Deleting tid::read_by_uid skipped'); + next(); + } } // Add new schema updates here // IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema IN LINE 22!!! From 2bc70079539e9179bed13137ad6a5a711d4422e7 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sat, 27 Sep 2014 18:09:25 -0400 Subject: [PATCH 4/4] removed sendUnread function --- src/topics/unread.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/topics/unread.js b/src/topics/unread.js index f1233b8eac..7777f82ae5 100644 --- a/src/topics/unread.js +++ b/src/topics/unread.js @@ -27,7 +27,11 @@ module.exports = function(Topics) { topics: [] }; - function sendUnreadTopics(tids, callback) { + Topics.getUnreadTids(uid, start, stop, function(err, tids) { + if (err) { + return callback(err); + } + if (!tids.length) { return callback(null, unreadTopics); } @@ -52,14 +56,6 @@ module.exports = function(Topics) { callback(null, unreadTopics); }); }); - } - - Topics.getUnreadTids(uid, start, stop, function(err, unreadTids) { - if (err) { - return callback(err); - } - - sendUnreadTopics(unreadTids, callback); }); };