From c3a9767bf6f5aee391066e8c3a75fb15d0a5fd2d Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sat, 28 Jun 2014 01:03:26 -0400 Subject: [PATCH] late night optimizations -isMemberOfSets returns true/false instead of 1/0 -when loading the posts of a topic only get the userdata for each user once, before this commit if a topic had 10 posts from 2 different users we were getting the user data for each user 5 times (drunk) -getVoteStatusByPostIDs and getFavouritesByPostIDs no longer make pids.length calls to the db, they use isMemberOfSets now -getUserInfoForPost renamed to getUserInfoForPosts and doesnt make uids.length calls to db, uses getMultipleUserFields instead --- src/database/mongo/sets.js | 2 +- src/database/redis/sets.js | 13 ++++++-- src/favourites.js | 28 ++++++++++++---- src/groups.js | 2 +- src/posts.js | 65 ++++++++++++++++++++------------------ src/topics.js | 44 ++++++++++++++------------ src/topics/create.js | 4 +-- src/topics/posts.js | 65 ++++++++++++++++++++++++-------------- src/topics/unread.js | 2 +- 9 files changed, 138 insertions(+), 87 deletions(-) diff --git a/src/database/mongo/sets.js b/src/database/mongo/sets.js index 73b4e6428b..e8d4f1a1f0 100644 --- a/src/database/mongo/sets.js +++ b/src/database/mongo/sets.js @@ -78,7 +78,7 @@ module.exports = function(db, module) { }); result = sets.map(function(set) { - return result.indexOf(set) !== -1 ? 1 : 0; + return result.indexOf(set) !== -1; }); callback(null, result); diff --git a/src/database/redis/sets.js b/src/database/redis/sets.js index 32c59a060a..10ef126b06 100644 --- a/src/database/redis/sets.js +++ b/src/database/redis/sets.js @@ -40,11 +40,20 @@ module.exports = function(redisClient, module) { module.isMemberOfSets = function(sets, value, callback) { var multi = redisClient.multi(); - for (var i = 0, ii = sets.length; i < ii; i++) { + for (var i = 0; i < sets.length; ++i) { multi.sismember(sets[i], value); } - multi.exec(callback); + multi.exec(function(err, results) { + if (err) { + return callback(err); + } + + for (var i=0; i=0; --i) { - if (parseInt(isMembers[i], 10) !== 1) { + if (!isMembers[i]) { groupData.splice(i, 1); } } diff --git a/src/posts.js b/src/posts.js index 519bb5453a..9c86ce4aec 100644 --- a/src/posts.js +++ b/src/posts.js @@ -188,43 +188,46 @@ var async = require('async'), }); }; - Posts.getUserInfoForPost = function(post, callback) { - user.getUserFields(post.uid, ['username', 'userslug', 'reputation', 'postcount', 'picture', 'signature', 'banned'], function(err, userData) { + Posts.getUserInfoForPosts = function(uids, callback) { + user.getMultipleUserFields(uids, ['uid', 'username', 'userslug', 'reputation', 'postcount', 'picture', 'signature', 'banned'], function(err, userData) { if (err) { return callback(err); } - var userInfo = { - username: userData.username || '[[global:guest]]', - userslug: userData.userslug || '', - reputation: userData.reputation || 0, - postcount: userData.postcount || 0, - banned: parseInt(userData.banned, 10) === 1, - picture: userData.picture || user.createGravatarURLFromEmail('') - }; + async.map(userData, function(userData, next) { + var userInfo = { + uid: userData.uid, + username: userData.username || '[[global:guest]]', + userslug: userData.userslug || '', + reputation: userData.reputation || 0, + postcount: userData.postcount || 0, + banned: parseInt(userData.banned, 10) === 1, + picture: userData.picture || user.createGravatarURLFromEmail('') + }; - async.parallel({ - signature: function(next) { - if (parseInt(meta.config.disableSignatures, 10) === 1) { - return next(); + async.parallel({ + signature: function(next) { + if (parseInt(meta.config.disableSignatures, 10) === 1) { + return next(); + } + postTools.parseSignature(userData.signature, next); + }, + customProfileInfo: function(next) { + plugins.fireHook('filter:posts.custom_profile_info', {profile: [], uid: userData.uid}, next); + }, + groups: function(next) { + groups.getUserGroups(userData.uid, next); } - postTools.parseSignature(userData.signature, next); - }, - customProfileInfo: function(next) { - plugins.fireHook('filter:posts.custom_profile_info', {profile: [], uid: post.uid, pid: post.pid}, next); - }, - groups: function(next) { - groups.getUserGroups(post.uid, next); - } - }, function(err, results) { - if (err) { - return callback(err); - } - userInfo.signature = results.signature; - userInfo.custom_profile_info = results.custom_profile_info; - userInfo.groups = results.groups; - callback(null, userInfo); - }); + }, function(err, results) { + if (err) { + return next(err); + } + userInfo.signature = results.signature; + userInfo.custom_profile_info = results.custom_profile_info; + userInfo.groups = results.groups; + next(null, userInfo); + }); + }, callback); }); }; diff --git a/src/topics.js b/src/topics.js index 52f84f575f..9157ab8f48 100644 --- a/src/topics.js +++ b/src/topics.js @@ -270,6 +270,9 @@ var async = require('async'), } async.parallel({ + mainPost: function(next) { + Topics.getMainPost(tid, uid, next); + }, posts: function(next) { Topics.getTopicPosts(tid, set, start, end, uid, reverse, next); }, @@ -284,26 +287,6 @@ var async = require('async'), }, tags: function(next) { Topics.getTopicTagsObjects(tid, next); - }, - mainPost: function(next) { - Topics.getTopicField(tid, 'mainPid', function(err, mainPid) { - if (err) { - return next(err); - } - if (!parseInt(mainPid, 10)) { - return next(null, []); - } - posts.getPostsByPids([mainPid], function(err, postData) { - if (err) { - return next(err); - } - if (!Array.isArray(postData) || !postData[0]) { - return next(null, []); - } - postData[0].index = 0; - Topics.addPostData(postData, uid, next); - }); - }); } }, function(err, results) { if (err) { @@ -325,6 +308,27 @@ var async = require('async'), }); }; + Topics.getMainPost = function(tid, uid, callback) { + Topics.getTopicField(tid, 'mainPid', function(err, mainPid) { + if (err) { + return callback(err); + } + if (!parseInt(mainPid, 10)) { + return callback(null, []); + } + posts.getPostsByPids([mainPid], function(err, postData) { + if (err) { + return callback(err); + } + if (!Array.isArray(postData) || !postData[0]) { + return callback(null, []); + } + postData[0].index = 0; + Topics.addPostData(postData, uid, callback); + }); + }); + }; + Topics.getTeasers = function(tids, callback) { if(!Array.isArray(tids)) { diff --git a/src/topics/create.js b/src/topics/create.js index cbaeec4947..ea3e56933c 100644 --- a/src/topics/create.js +++ b/src/topics/create.js @@ -224,10 +224,10 @@ module.exports = function(Topics) { }, function(result, next) { Topics.pushUnreadCount(); - posts.getUserInfoForPost(postData, next); + posts.getUserInfoForPosts([postData.uid], next); }, function(userInfo, next) { - postData.user = userInfo; + postData.user = userInfo[0]; Topics.getTopicFields(tid, ['tid', 'title', 'slug'], next); }, function(topicData, next) { diff --git a/src/topics/posts.js b/src/topics/posts.js index b787c3956c..00903373a8 100644 --- a/src/topics/posts.js +++ b/src/topics/posts.js @@ -42,6 +42,7 @@ module.exports = function(Topics) { }; Topics.addPostData = function(postData, uid, callback) { + var st = process.hrtime(); var pids = postData.map(function(post) { return post.pid; }); @@ -54,27 +55,43 @@ module.exports = function(Topics) { favourites.getVoteStatusByPostIDs(pids, uid, next); }, userData: function(next) { - async.each(postData, function(post, next) { - async.parallel({ - editor: function(next) { - if (!post.editor) { - return next(); - } - user.getUserFields(post.editor, ['username', 'userslug'], next); - }, - user: function(next) { - posts.getUserInfoForPost(post, next); - } - }, function(err, results) { - if (err) { - return next(err); - } - - post.user = results.user; - post.editor = results.editor; - next(); + var uids = []; + for(var i=0; i