From 53d29e29af8f5347145fdd2c51c028e76f10334d Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sun, 18 Oct 2015 18:30:17 -0400 Subject: [PATCH] performance improvements store parsed category description removed mongo _key from returns dont get category teaser for parent --- src/categories/data.js | 14 +-- src/categories/recentreplies.js | 172 +++++++++++++++----------------- src/categories/update.js | 11 ++ src/controllers/categories.js | 7 +- src/database/mongo/helpers.js | 1 + src/database/mongo/sorted.js | 2 +- 6 files changed, 105 insertions(+), 102 deletions(-) diff --git a/src/categories/data.js b/src/categories/data.js index c844e632ad..cccacc8a68 100644 --- a/src/categories/data.js +++ b/src/categories/data.js @@ -53,17 +53,11 @@ module.exports = function(Categories) { } if (category.description) { - plugins.fireHook('filter:parse.raw', category.description, function(err, parsedDescription) { - if (err) { - return callback(err); - } - category.descriptionParsed = parsedDescription; - category.description = validator.escape(category.description); - callback(null, category); - }); - } else { - callback(null, category); + category.description = validator.escape(category.description); + category.descriptionParsed = category.descriptionParsed || category.description; } + + callback(null, category); } Categories.getCategoryField = function(cid, field, callback) { diff --git a/src/categories/recentreplies.js b/src/categories/recentreplies.js index d7a2a165a0..84cb005de0 100644 --- a/src/categories/recentreplies.js +++ b/src/categories/recentreplies.js @@ -3,14 +3,13 @@ var async = require('async'), winston = require('winston'), + validator = require('validator'), _ = require('underscore'), - meta = require('../meta'), db = require('../database'), posts = require('../posts'), topics = require('../topics'), - privileges = require('../privileges'), - plugins = require('../plugins'); + privileges = require('../privileges'); module.exports = function(Categories) { Categories.getRecentReplies = function(cid, uid, count, callback) { @@ -38,25 +37,21 @@ module.exports = function(Categories) { async.waterfall([ function(next) { - async.map(categoryData, getRecentTopicPids, next); + async.map(categoryData, getRecentTopicTids, next); }, function(results, next) { - var pids = _.flatten(results); + var tids = _.flatten(results); - pids = pids.filter(function(pid, index, array) { - return !!pid && array.indexOf(pid) === index; + tids = tids.filter(function(tid, index, array) { + return !!tid && array.indexOf(tid) === index; }); - privileges.posts.filter('read', pids, uid, next); + privileges.topics.filterTids('read', tids, uid, next); }, - function(pids, next) { - if (meta.config.teaserPost === 'first') { - getMainPosts(pids, uid, next); - } else { - posts.getPostSummaryByPids(pids, uid, {stripTags: true}, next); - } + function(tids, next) { + getTopics(tids, next); }, - function(posts, next) { - assignPostsToCategories(categoryData, posts); + function(topics, next) { + assignTopicsToCategories(categoryData, topics); bubbleUpChildrenPosts(categoryData); @@ -65,29 +60,86 @@ module.exports = function(Categories) { ], callback); }; - function getMainPosts(pids, uid, callback) { + function getRecentTopicTids(category, callback) { + var count = parseInt(category.numRecentReplies, 10); + if (!count) { + return callback(null, []); + } + + if (count === 1) { + async.waterfall([ + function (next) { + db.getSortedSetRevRange('cid:' + category.cid + ':pids', 0, 0, next); + }, + function (pid, next) { + posts.getPostField(pid, 'tid', next); + }, + function (tid, next) { + next(null, [tid]); + } + ], callback); + return; + } + + async.parallel({ + pinnedTids: function(next) { + db.getSortedSetRevRangeByScore('cid:' + category.cid + ':tids', 0, -1, '+inf', Date.now(), next); + }, + tids: function(next) { + db.getSortedSetRevRangeByScore('cid:' + category.cid + ':tids', 0, Math.max(1, count), Date.now(), 0, next); + } + }, function(err, results) { + if (err) { + return callback(err); + } + + results.tids = results.tids.concat(results.pinnedTids); + + callback(null, results.tids); + }); + } + + function getTopics(tids, callback) { + var topicData; async.waterfall([ - function(next) { - var keys = pids.map(function(pid) { - return 'post:' + pid; - }); - db.getObjectsFields(keys, ['tid'], next); + function (next) { + topics.getTopicsFields(tids, ['tid', 'mainPid', 'slug', 'title', 'teaserPid', 'cid', 'postcount'], next); }, - function(posts, next) { - var keys = posts.map(function(post) { - return 'topic:' + post.tid; + function (_topicData, next) { + topicData = _topicData; + topicData.forEach(function(topic) { + topic.teaserPid = topic.teaserPid || topic.mainPid; }); - db.getObjectsFields(keys, ['mainPid'], next); + + topics.getTeasers(topicData, next); }, - function(topics, next) { - var mainPids = topics.map(function(topic) { - return topic.mainPid; + function (teasers, next) { + teasers.forEach(function(teaser, index) { + if (teaser) { + teaser.cid = topicData[index].cid; + teaser.tid = teaser.uid = teaser.user.uid = undefined; + teaser.topic = { + slug: topicData[index].slug, + title: validator.escape(topicData[index].title) + }; + } }); - posts.getPostSummaryByPids(mainPids, uid, {stripTags: true}, next); + teasers = teasers.filter(Boolean); + next(null, teasers); } ], callback); } + function assignTopicsToCategories(categories, topics) { + categories.forEach(function(category) { + category.posts = topics.filter(function(topic) { + return topic.cid && parseInt(topic.cid, 10) === parseInt(category.cid, 10); + }).sort(function(a, b) { + return b.pid - a.pid; + }).slice(0, parseInt(category.numRecentReplies, 10)); + }); + } + function bubbleUpChildrenPosts(categoryData) { categoryData.forEach(function(category) { if (category.posts.length) { @@ -97,7 +149,7 @@ module.exports = function(Categories) { getPostsRecursive(category, posts); posts.sort(function(a, b) { - return b.timestamp - a.timestamp; + return b.pid - a.pid; }); if (posts.length) { category.posts = [posts[0]]; @@ -115,64 +167,6 @@ module.exports = function(Categories) { }); } - function assignPostsToCategories(categories, posts) { - categories.forEach(function(category) { - category.posts = posts.filter(function(post) { - return post.category && (parseInt(post.category.cid, 10) === parseInt(category.cid, 10) || - parseInt(post.category.parentCid, 10) === parseInt(category.cid, 10)); - }).sort(function(a, b) { - return b.timestamp - a.timestamp; - }).slice(0, parseInt(category.numRecentReplies, 10)); - }); - } - - function getRecentTopicPids(category, callback) { - var count = parseInt(category.numRecentReplies, 10); - if (!count) { - return callback(null, []); - } - - db.getSortedSetRevRange('cid:' + category.cid + ':pids', 0, 0, function(err, pids) { - if (err || !Array.isArray(pids) || !pids.length) { - return callback(err, []); - } - - if (count === 1) { - return callback(null, pids); - } - - async.parallel({ - pinnedTids: function(next) { - db.getSortedSetRevRangeByScore('cid:' + category.cid + ':tids', 0, -1, '+inf', Date.now(), next); - }, - tids: function(next) { - db.getSortedSetRevRangeByScore('cid:' + category.cid + ':tids', 0, Math.max(0, count), Date.now(), 0, next); - } - }, function(err, results) { - if (err) { - return callback(err); - } - - results.tids = results.tids.concat(results.pinnedTids); - - async.map(results.tids, topics.getLatestUndeletedPid, function(err, topicPids) { - if (err) { - return callback(err); - } - - pids = pids.concat(topicPids).filter(function(pid, index, array) { - return !!pid && array.indexOf(pid) === index; - }).sort(function(a, b) { - return b - a; - }).slice(0, count); - - callback(null, pids); - }); - }); - }); - } - - Categories.moveRecentReplies = function(tid, oldCid, cid) { updatePostCount(tid, oldCid, cid); topics.getPids(tid, function(err, pids) { diff --git a/src/categories/update.js b/src/categories/update.js index 4ebfda2f11..ffd24e7d98 100644 --- a/src/categories/update.js +++ b/src/categories/update.js @@ -68,6 +68,8 @@ module.exports = function(Categories) { if (key === 'order') { updateOrder(cid, value, callback); + } else if (key === 'description') { + parseDescription(cid, value, callback); } else { callback(); } @@ -119,4 +121,13 @@ module.exports = function(Categories) { }); } + function parseDescription(cid, description, callback) { + plugins.fireHook('filter:parse.raw', description, function(err, parsedDescription) { + if (err) { + return callback(err); + } + Categories.setCategoryField(cid, 'descriptionParsed', parsedDescription, callback); + }); + } + }; diff --git a/src/controllers/categories.js b/src/controllers/categories.js index 59068434ad..b2d36b91e8 100644 --- a/src/controllers/categories.js +++ b/src/controllers/categories.js @@ -69,7 +69,7 @@ categoriesController.list = function(req, res, next) { if (category && Array.isArray(category.posts) && category.posts.length) { category.teaser = { url: nconf.get('relative_path') + '/topic/' + category.posts[0].topic.slug + '/' + category.posts[0].index, - timestampISO: category.posts[0].relativeTime + timestampISO: category.posts[0].timestamp }; } }); @@ -203,8 +203,11 @@ categoriesController.get = function(req, res, callback) { }); }, function(categoryData, next) { + if (!categoryData.children.length) { + return next(null, categoryData); + } var allCategories = []; - categories.flattenCategories(allCategories, [categoryData]); + categories.flattenCategories(allCategories, categoryData.children); categories.getRecentTopicReplies(allCategories, req.uid, function(err) { next(err, categoryData); }); diff --git a/src/database/mongo/helpers.js b/src/database/mongo/helpers.js index 0be79b5388..db84dbb369 100644 --- a/src/database/mongo/helpers.js +++ b/src/database/mongo/helpers.js @@ -6,6 +6,7 @@ helpers.toMap = function(data) { var map = {}; for (var i = 0; i