From c7128bcc989bac77bb810f3aaa2964892b59dd53 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sat, 14 Mar 2015 23:00:24 -0400 Subject: [PATCH 1/8] part 1 of search change --- src/database/mongo.js | 2 + src/database/mongo/main.js | 38 +++++++--- src/database/redis.js | 12 ++-- src/database/redis/main.js | 22 ++---- src/search.js | 141 ++++++++++++++++++++----------------- src/user.js | 6 +- 6 files changed, 125 insertions(+), 96 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 0e69c12380..9e4399b031 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -112,6 +112,8 @@ createIndex('search', {content:'text'}, {background:true}); createIndex('search', {key: 1, id: 1}, {background:true}); + createIndex('search', {cid: 1}, {background:true}); + createIndex('search', {uid: 1}, {background:true}); if (typeof callback === 'function') { callback(); diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js index 029b8f1fee..a40567e2a6 100644 --- a/src/database/mongo/main.js +++ b/src/database/mongo/main.js @@ -5,15 +5,19 @@ var winston = require('winston'); module.exports = function(db, module) { var helpers = module.helpers.mongo; - module.searchIndex = function(key, content, id, callback) { + module.searchIndex = function(key, data, id, callback) { callback = callback || function() {}; - var data = { + var setData = { id: id, - key: key, - content: content + key: key }; + for(var field in data) { + if (data.hasOwnProperty(field)) { + setData[field] = data[field]; + } + } - db.collection('search').update({key:key, id:id}, {$set:data}, {upsert:true, w: 1}, function(err) { + db.collection('search').update({key: key, id: id}, {$set: setData}, {upsert:true, w: 1}, function(err) { if(err) { winston.error('Error indexing ' + err.message); } @@ -21,13 +25,29 @@ module.exports = function(db, module) { }); }; - module.search = function(key, term, limit, callback) { - db.collection('search').find({ $text: { $search: term }, key: key}, {limit: limit}).toArray(function(err, results) { - if(err) { + module.search = function(key, data, limit, callback) { + var searchQuery = { + key: key + }; + + if (data.content) { + searchQuery.$text = {$search: data.content}; + } + + if (data.cid) { + searchQuery.cid = data.cid; + } + + if (data.uid) { + searchQuery.uid = data.uid; + } + + db.collection('search').find(searchQuery, {limit: limit}).toArray(function(err, results) { + if (err) { return callback(err); } - if(!results || !results.length) { + if (!results || !results.length) { return callback(null, []); } diff --git a/src/database/redis.js b/src/database/redis.js index 727705c130..716a8fd715 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -9,7 +9,7 @@ utils = require('./../../public/src/utils.js'), redis, connectRedis, - reds, + redisSearch, redisClient, postSearch, topicSearch; @@ -41,7 +41,7 @@ try { redis = require('redis'); connectRedis = require('connect-redis')(session); - reds = require('reds'); + redisSearch = require('redisearch'); } catch (err) { winston.error('Unable to initialize Redis! Is Redis installed? Error :' + err.message); process.exit(); @@ -56,12 +56,8 @@ ttl: 60 * 60 * 24 * 14 }); - reds.createClient = function () { - return reds.client || (reds.client = redisClient); - }; - - module.postSearch = reds.createSearch('nodebbpostsearch'); - module.topicSearch = reds.createSearch('nodebbtopicsearch'); + module.postSearch = redisSearch.createSearch('nodebbpostsearch', redisClient); + module.topicSearch = redisSearch.createSearch('nodebbtopicsearch', redisClient); require('./redis/main')(redisClient, module); require('./redis/hash')(redisClient, module); diff --git a/src/database/redis/main.js b/src/database/redis/main.js index 094fa3b695..9ed956f0b3 100644 --- a/src/database/redis/main.js +++ b/src/database/redis/main.js @@ -1,27 +1,19 @@ "use strict"; module.exports = function(redisClient, module) { - module.searchIndex = function(key, content, id, callback) { + module.searchIndex = function(key, data, id, callback) { if (key === 'post') { - module.postSearch.index(content, id, callback); + module.postSearch.index(data, id, callback); } else if(key === 'topic') { - module.topicSearch.index(content, id, callback); + module.topicSearch.index(data, id, callback); } }; - module.search = function(key, term, limit, callback) { - function search(searchObj, callback) { - searchObj - .query(term) - .between(0, limit - 1) - .type('or') - .end(callback); - } - - if(key === 'post') { - search(module.postSearch, callback); + module.search = function(key, data, limit, callback) { + if (key === 'post') { + module.postSearch.query(data, 0, limit - 1, callback); } else if(key === 'topic') { - search(module.topicSearch, callback); + module.topicSearch.query(data, 0, limit - 1, callback); } }; diff --git a/src/search.js b/src/search.js index e44dedc216..a478d7f8e9 100644 --- a/src/search.js +++ b/src/search.js @@ -43,7 +43,7 @@ search.search = function(data, callback) { }; if (searchIn === 'posts' || searchIn === 'titles' || searchIn === 'titlesposts') { - searchInContent(query, data, done); + searchInContent(data, done); } else if (searchIn === 'users') { searchInUsers(query, data.uid, done); } else if (searchIn === 'tags') { @@ -53,77 +53,80 @@ search.search = function(data, callback) { } }; -function searchInContent(query, data, callback) { +function searchInContent(data, callback) { data.uid = data.uid || 0; async.parallel({ - pids: function(next) { - if (data.searchIn === 'posts' || data.searchIn === 'titlesposts') { - search.searchQuery('post', query, next); - } else { - next(null, []); - } - }, - tids: function(next) { - if (data.searchIn === 'titles' || data.searchIn === 'titlesposts') { - search.searchQuery('topic', query, next); - } else { - next(null, []); - } + searchCids: function(next) { + getSearchCids(data, next); }, - searchCategories: function(next) { - getSearchCategories(data, next); + searchUids: function(next) { + getSearchUids(data, next); } - }, function (err, results) { + }, function(err, results) { if (err) { return callback(err); } - var matchCount = 0; - if (!results || (!results.pids.length && !results.tids.length)) { - return callback(null, {matches: [], matchCount: matchCount}); - } - - async.waterfall([ - function(next) { - topics.getMainPids(results.tids, next); - }, - function(mainPids, next) { - results.pids = mainPids.concat(results.pids).filter(function(pid, index, array) { - return pid && array.indexOf(pid) === index; - }); - - privileges.posts.filter('read', results.pids, data.uid, next); - }, - function(pids, next) { - filterAndSort(pids, data, results.searchCategories, next); - }, - function(pids, next) { - matchCount = pids.length; - if (data.page) { - var start = Math.max(0, (data.page - 1)) * 10; - pids = pids.slice(start, start + 10); + async.parallel({ + pids: function(next) { + if (data.searchIn === 'posts' || data.searchIn === 'titlesposts') { + search.searchQuery('post', data.query, results.searchCids, results.searchUids, next); + } else { + next(null, []); } - - posts.getPostSummaryByPids(pids, data.uid, {stripTags: true, parse: false}, next); }, - function(posts, next) { - next(null, {matches: posts, matchCount: matchCount}); + tids: function(next) { + if (data.searchIn === 'titles' || data.searchIn === 'titlesposts') { + search.searchQuery('topic', data.query, results.searchCids, results.searchUids, next); + } else { + next(null, []); + } + } + }, function (err, results) { + if (err) { + return callback(err); + } +console.log(results) + var matchCount = 0; + if (!results || (!results.pids.length && !results.tids.length)) { + return callback(null, {matches: [], matchCount: matchCount}); } - ], callback); + + async.waterfall([ + function(next) { + topics.getMainPids(results.tids, next); + }, + function(mainPids, next) { + results.pids = mainPids.concat(results.pids).filter(function(pid, index, array) { + return pid && array.indexOf(pid) === index; + }); + + privileges.posts.filter('read', results.pids, data.uid, next); + }, + function(pids, next) { + filterAndSort(pids, data, results.searchCategories, next); + }, + function(pids, next) { + matchCount = pids.length; + if (data.page) { + var start = Math.max(0, (data.page - 1)) * 10; + pids = pids.slice(start, start + 10); + } + + posts.getPostSummaryByPids(pids, data.uid, {stripTags: true, parse: false}, next); + }, + function(posts, next) { + next(null, {matches: posts, matchCount: matchCount}); + } + ], callback); + }); }); } -function filterAndSort(pids, data, searchCategories, callback) { +function filterAndSort(pids, data, callback) { async.parallel({ posts: function(next) { - getMatchedPosts(pids, data, searchCategories, next); - }, - postedByUid: function(next) { - if (data.postedBy) { - user.getUidByUsername(data.postedBy, next); - } else { - next(); - } + getMatchedPosts(pids, data, next); } }, function(err, results) { if (err) { @@ -134,8 +137,6 @@ function filterAndSort(pids, data, searchCategories, callback) { } var posts = results.posts.filter(Boolean); - posts = filterByUser(posts, results.postedByUid); - posts = filterByCategories(posts, searchCategories); posts = filterByPostcount(posts, data.replies, data.repliesFilter); posts = filterByTimerange(posts, data.timeRange, data.timeFilter); @@ -149,7 +150,7 @@ function filterAndSort(pids, data, searchCategories, callback) { }); } -function getMatchedPosts(pids, data, searchCategories, callback) { +function getMatchedPosts(pids, data, callback) { var postFields = ['pid', 'tid', 'timestamp']; var topicFields = []; var categoryFields = []; @@ -158,7 +159,7 @@ function getMatchedPosts(pids, data, searchCategories, callback) { postFields.push('uid'); } - if (searchCategories.length || (data.sortBy && data.sortBy.startsWith('category.'))) { + if (data.sortBy && data.sortBy.startsWith('category.')) { topicFields.push('cid'); } @@ -389,7 +390,7 @@ function sortPosts(posts, data) { } } -function getSearchCategories(data, callback) { +function getSearchCids(data, callback) { if (!Array.isArray(data.categories) || !data.categories.length || data.categories.indexOf('all') !== -1) { return callback(null, []); } @@ -439,6 +440,18 @@ function getChildrenCids(cids, uid, callback) { }); } +function getSearchUids(data, callback) { + if (data.postedBy) { + if (Array.isArray(data.postedBy)) { + user.getUidsByUsernames(data.postedBy, callback); + } else { + user.getUidByUsername([data.postedBy], callback); + } + } else { + callback(null, []); + } +} + function searchInUsers(query, uid, callback) { user.search({query: query, uid: uid}, function(err, results) { if (err) { @@ -458,10 +471,12 @@ function searchInTags(query, callback) { }); } -search.searchQuery = function(index, query, callback) { +search.searchQuery = function(index, content, cids, uids, callback) { plugins.fireHook('filter:search.query', { index: index, - query: query + content: content, + cids: cids, + uids: uids }, callback); }; diff --git a/src/user.js b/src/user.js index 109e67a3f0..bd0eed4ef4 100644 --- a/src/user.js +++ b/src/user.js @@ -158,7 +158,7 @@ var async = require('async'), if (now - parseInt(userOnlineTime, 10) < 300000) { return callback(); } - db.sortedSetAdd('users:online', now, uid, next); + db.sortedSetAdd('users:online', now, uid, next); }, function(next) { topics.pushUnreadCount(uid); @@ -334,6 +334,10 @@ var async = require('async'), db.getObjectField('username:uid', username, callback); }; + User.getUidsByUsernames = function(usernames, callback) { + db.getObjectFields('username:uid', usernames, callback); + }; + User.getUidByUserslug = function(userslug, callback) { if (!userslug) { return callback(); From e2196af8ab38589504654f3b2cbe34f6aecd0ec2 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sun, 15 Mar 2015 00:12:52 -0400 Subject: [PATCH 2/8] part 2 of search changes --- src/database/mongo/main.js | 4 +++- src/database/redis/main.js | 27 ++++++++++----------- src/search.js | 48 ++++++++------------------------------ 3 files changed, 25 insertions(+), 54 deletions(-) diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js index a40567e2a6..6302d76fb2 100644 --- a/src/database/mongo/main.js +++ b/src/database/mongo/main.js @@ -64,7 +64,9 @@ module.exports = function(db, module) { if (!id) { return callback(); } - db.collection('search').remove({key: key, id: id}, callback); + db.collection('search').remove({key: key, id: id}, function(err, res) { + callback(err); + }); }; module.flushdb = function(callback) { diff --git a/src/database/redis/main.js b/src/database/redis/main.js index 9ed956f0b3..b09574e10f 100644 --- a/src/database/redis/main.js +++ b/src/database/redis/main.js @@ -2,19 +2,17 @@ module.exports = function(redisClient, module) { module.searchIndex = function(key, data, id, callback) { - if (key === 'post') { - module.postSearch.index(data, id, callback); - } else if(key === 'topic') { - module.topicSearch.index(data, id, callback); - } + var method = key === 'post' ? module.postSearch : module.topicSearch; + + method.index(data, id, function(err, res) { + callback(err); + }); }; module.search = function(key, data, limit, callback) { - if (key === 'post') { - module.postSearch.query(data, 0, limit - 1, callback); - } else if(key === 'topic') { - module.topicSearch.query(data, 0, limit - 1, callback); - } + var method = key === 'post' ? module.postSearch : module.topicSearch; + + method.query(data, 0, limit - 1, callback); }; module.searchRemove = function(key, id, callback) { @@ -22,12 +20,11 @@ module.exports = function(redisClient, module) { if (!id) { return callback(); } + var method = key === 'post' ? module.postSearch : module.topicSearch; - if (key === 'post') { - module.postSearch.remove(id, callback); - } else if(key === 'topic') { - module.topicSearch.remove(id, callback); - } + method.remove(id, function(err, res) { + callback(err); + }); }; module.flushdb = function(callback) { diff --git a/src/search.js b/src/search.js index a478d7f8e9..8315312bec 100644 --- a/src/search.js +++ b/src/search.js @@ -104,7 +104,7 @@ console.log(results) privileges.posts.filter('read', results.pids, data.uid, next); }, function(pids, next) { - filterAndSort(pids, data, results.searchCategories, next); + filterAndSort(pids, data, next); }, function(pids, next) { matchCount = pids.length; @@ -124,18 +124,15 @@ console.log(results) } function filterAndSort(pids, data, callback) { - async.parallel({ - posts: function(next) { - getMatchedPosts(pids, data, next); - } - }, function(err, results) { + getMatchedPosts(pids, data, function(err, posts) { if (err) { return callback(err); } - if (!results.posts) { + + if (!Array.isArray(posts) || !posts.length) { return callback(null, pids); } - var posts = results.posts.filter(Boolean); + posts = posts.filter(Boolean); posts = filterByPostcount(posts, data.replies, data.repliesFilter); posts = filterByTimerange(posts, data.timeRange, data.timeFilter); @@ -155,20 +152,14 @@ function getMatchedPosts(pids, data, callback) { var topicFields = []; var categoryFields = []; - if (data.postedBy) { - postFields.push('uid'); - } - - if (data.sortBy && data.sortBy.startsWith('category.')) { - topicFields.push('cid'); - } - if (data.replies) { topicFields.push('postcount'); } if (data.sortBy) { - if (data.sortBy.startsWith('topic.')) { + if (data.sortBy.startsWith('category')) { + topicFields.push('cid'); + } else if (data.sortBy.startsWith('topic.')) { topicFields.push(data.sortBy.split('.')[1]); } else if (data.sortBy.startsWith('user.')) { postFields.push('uid'); @@ -281,25 +272,6 @@ function getMatchedPosts(pids, data, callback) { ], callback); } -function filterByUser(posts, postedByUid) { - if (postedByUid) { - postedByUid = parseInt(postedByUid, 10); - posts = posts.filter(function(post) { - return parseInt(post.uid, 10) === postedByUid; - }); - } - return posts; -} - -function filterByCategories(posts, searchCategories) { - if (searchCategories.length) { - posts = posts.filter(function(post) { - return post.topic && searchCategories.indexOf(post.topic.cid) !== -1; - }); - } - return posts; -} - function filterByPostcount(posts, postCount, repliesFilter) { postCount = parseInt(postCount, 10); if (postCount) { @@ -475,8 +447,8 @@ search.searchQuery = function(index, content, cids, uids, callback) { plugins.fireHook('filter:search.query', { index: index, content: content, - cids: cids, - uids: uids + cid: cids, + uid: uids }, callback); }; From 31da62ab2d78832fd987012892fc0501e4f23b03 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sun, 15 Mar 2015 00:13:26 -0400 Subject: [PATCH 3/8] removed console.log --- src/search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/search.js b/src/search.js index 8315312bec..257ce6f352 100644 --- a/src/search.js +++ b/src/search.js @@ -86,7 +86,7 @@ function searchInContent(data, callback) { if (err) { return callback(err); } -console.log(results) + var matchCount = 0; if (!results || (!results.pids.length && !results.tids.length)) { return callback(null, {matches: [], matchCount: matchCount}); From 2b7744f9050151db0160483ee52983ce3da15a77 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sun, 15 Mar 2015 00:24:08 -0400 Subject: [PATCH 4/8] fire action:topic.move after topic cid is changed --- src/threadTools.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/threadTools.js b/src/threadTools.js index 96d8198291..109a4215ce 100644 --- a/src/threadTools.js +++ b/src/threadTools.js @@ -183,13 +183,16 @@ var async = require('async'), categories.moveRecentReplies(tid, oldCid, cid); - topics.setTopicField(tid, 'cid', cid, callback); - - plugins.fireHook('action:topic.move', { - tid: tid, - fromCid: oldCid, - toCid: cid, - uid: uid + topics.setTopicField(tid, 'cid', cid, function(err) { + if (err) { + return callback(err); + } + plugins.fireHook('action:topic.move', { + tid: tid, + fromCid: oldCid, + toCid: cid, + uid: uid + }); }); }); }; From 78c65aee0518c23b3a393ebd353b6c4f1ed9a7c8 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sun, 15 Mar 2015 01:12:13 -0400 Subject: [PATCH 5/8] even more search changes --- src/categories/topics.js | 50 ++++++++++-------------- src/database/mongo/main.js | 4 +- src/postTools.js | 21 ++++++++-- src/posts/create.js | 8 +++- src/posts/delete.js | 79 ++++++++++++++++---------------------- src/threadTools.js | 2 +- 6 files changed, 81 insertions(+), 83 deletions(-) diff --git a/src/categories/topics.js b/src/categories/topics.js index 50dba67bbd..805ab30c4a 100644 --- a/src/categories/topics.js +++ b/src/categories/topics.js @@ -74,37 +74,29 @@ module.exports = function(Categories) { }); }; - Categories.onNewPostMade = function(postData, callback) { - topics.getTopicFields(postData.tid, ['cid', 'pinned'], function(err, topicData) { - if (err) { - return callback(err); - } - - if (!topicData || !topicData.cid) { - return callback(); - } - - var cid = topicData.cid; + Categories.onNewPostMade = function(cid, pinned, postData, callback) { + if (!cid || !postData) { + return callback(); + } - async.parallel([ - function(next) { - db.sortedSetAdd('cid:' + cid + ':pids', postData.timestamp, postData.pid, next); - }, - function(next) { - db.incrObjectField('category:' + cid, 'post_count', next); - }, - function(next) { - if (parseInt(topicData.pinned, 10) === 1) { - next(); - } else { - db.sortedSetAdd('cid:' + cid + ':tids', postData.timestamp, postData.tid, next); - } - }, - function(next) { - db.sortedSetIncrBy('cid:' + cid + ':tids:posts', 1, postData.tid, next); + async.parallel([ + function(next) { + db.sortedSetAdd('cid:' + cid + ':pids', postData.timestamp, postData.pid, next); + }, + function(next) { + db.incrObjectField('category:' + cid, 'post_count', next); + }, + function(next) { + if (parseInt(pinned, 10) === 1) { + next(); + } else { + db.sortedSetAdd('cid:' + cid + ':tids', postData.timestamp, postData.tid, next); } - ], callback); - }); + }, + function(next) { + db.sortedSetIncrBy('cid:' + cid + ':tids:posts', 1, postData.tid, next); + } + ], callback); }; }; diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js index 6302d76fb2..913696aa9c 100644 --- a/src/database/mongo/main.js +++ b/src/database/mongo/main.js @@ -12,8 +12,8 @@ module.exports = function(db, module) { key: key }; for(var field in data) { - if (data.hasOwnProperty(field)) { - setData[field] = data[field]; + if (data.hasOwnProperty(field) && data[field]) { + setData[field] = data[field].toString(); } } diff --git a/src/postTools.js b/src/postTools.js index 3e9712699e..647d796841 100644 --- a/src/postTools.js +++ b/src/postTools.js @@ -19,7 +19,7 @@ var winston = require('winston'), var cache = LRU({ max: 1048576, - length: function (n) { return n.length }, + length: function (n) { return n.length; }, maxAge: 1000 * 60 * 60 }); @@ -63,22 +63,32 @@ var cache = LRU({ }, topic: function(next) { var tid = postData.tid; - posts.isMain(data.pid, function(err, isMainPost) { + async.parallel({ + cid: function(next) { + topics.getTopicField(tid, 'cid', next); + }, + isMain: function(next) { + posts.isMain(data.pid, next); + } + }, function(err, results) { if (err) { return next(err); } options.tags = options.tags || []; - if (!isMainPost) { + if (!results.isMain) { return next(null, { tid: tid, + cid: results.cid, isMainPost: false }); } var topicData = { tid: tid, + cid: results.cid, + uid: postData.uid, mainPid: data.pid, title: title, slug: tid + '/' + utils.slugify(title) @@ -98,8 +108,10 @@ var cache = LRU({ topics.getTopicTagsObjects(tid, function(err, tags) { next(err, { tid: tid, + cid: results.cid, + uid: postData.uid, title: validator.escape(title), - isMainPost: isMainPost, + isMainPost: results.isMain, tags: tags }); }); @@ -114,6 +126,7 @@ var cache = LRU({ if (err) { return callback(err); } + postData.cid = results.topic.cid; results.content = results.postData.content; plugins.fireHook('action:post.edit', postData); diff --git a/src/posts/create.js b/src/posts/create.js index a235b2f3f0..1bce6e69c7 100644 --- a/src/posts/create.js +++ b/src/posts/create.js @@ -68,7 +68,13 @@ module.exports = function(Posts) { topics.onNewPostMade(postData, next); }, function(next) { - categories.onNewPostMade(postData, next); + topics.getTopicFields(tid, ['cid', 'pinned'], function(err, topicData) { + if (err) { + return next(err); + } + postData.cid = topicData.cid; + categories.onNewPostMade(topicData.cid, topicData.pinned, postData, next); + }); }, function(next) { db.sortedSetAdd('posts:pid', timestamp, postData.pid, next); diff --git a/src/posts/delete.js b/src/posts/delete.js index bd130ade46..f47ff28e88 100644 --- a/src/posts/delete.js +++ b/src/posts/delete.js @@ -9,16 +9,19 @@ var async = require('async'), module.exports = function(Posts) { Posts.delete = function(pid, callback) { - Posts.setPostField(pid, 'deleted', 1, function(err) { - if (err) { - return callback(err); - } - - Posts.getPostFields(pid, ['pid', 'tid', 'uid', 'timestamp'], function(err, postData) { - if (err) { - return callback(err); - } - + var postData; + async.waterfall([ + function(next) { + Posts.setPostField(pid, 'deleted', 1, next); + }, + function(next) { + Posts.getPostFields(pid, ['pid', 'tid', 'uid', 'timestamp'], next); + }, + function(_post, next) { + postData = _post; + topics.getTopicField(_post.tid, 'cid', next); + }, + function(cid, next) { plugins.fireHook('action:post.delete', pid); async.parallel([ @@ -26,7 +29,7 @@ module.exports = function(Posts) { updateTopicTimestamp(postData.tid, next); }, function(next) { - removeFromCategoryRecentPosts(pid, postData.tid, next); + db.sortedSetRemove('cid:' + cid + ':pids', pid, next); }, function(next) { Posts.dismissFlag(pid, next); @@ -34,21 +37,25 @@ module.exports = function(Posts) { ], function(err) { callback(err, postData); }); - }); - }); + } + ], callback); }; Posts.restore = function(pid, callback) { - Posts.setPostField(pid, 'deleted', 0, function(err) { - if (err) { - return callback(err); - } - - Posts.getPostFields(pid, ['pid', 'tid', 'uid', 'content', 'timestamp'], function(err, postData) { - if (err) { - return callback(err); - } - + var postData; + async.waterfall([ + function(next) { + Posts.setPostField(pid, 'deleted', 0, next); + }, + function(next) { + Posts.getPostFields(pid, ['pid', 'tid', 'uid', 'content', 'timestamp'], next); + }, + function(_post, next) { + postData = _post; + topics.getTopicField(_post.tid, 'cid', next); + }, + function(cid, next) { + postData.cid = cid; plugins.fireHook('action:post.restore', postData); async.parallel([ @@ -56,13 +63,13 @@ module.exports = function(Posts) { updateTopicTimestamp(postData.tid, next); }, function(next) { - addToCategoryRecentPosts(pid, postData.tid, postData.timestamp, next); + db.sortedSetAdd('cid:' + cid + ':pids', postData.timestamp, pid, next); } ], function(err) { callback(err, postData); }); - }); - }); + } + ], callback); }; function updateTopicTimestamp(tid, callback) { @@ -84,26 +91,6 @@ module.exports = function(Posts) { }); } - function removeFromCategoryRecentPosts(pid, tid, callback) { - topics.getTopicField(tid, 'cid', function(err, cid) { - if (err) { - return callback(err); - } - - db.sortedSetRemove('cid:' + cid + ':pids', pid, callback); - }); - } - - function addToCategoryRecentPosts(pid, tid, timestamp, callback) { - topics.getTopicField(tid, 'cid', function(err, cid) { - if (err) { - return callback(err); - } - - db.sortedSetAdd('cid:' + cid + ':pids', timestamp, pid, callback); - }); - } - Posts.purge = function(pid, callback) { Posts.exists(pid, function(err, exists) { if (err || !exists) { diff --git a/src/threadTools.js b/src/threadTools.js index 109a4215ce..76ea84c9df 100644 --- a/src/threadTools.js +++ b/src/threadTools.js @@ -21,7 +21,7 @@ var async = require('async'), }; function toggleDelete(tid, uid, isDelete, callback) { - topics.getTopicFields(tid, ['tid', 'cid', 'deleted', 'title', 'mainPid'], function(err, topicData) { + topics.getTopicFields(tid, ['tid', 'cid', 'uid', 'deleted', 'title', 'mainPid'], function(err, topicData) { if (err) { return callback(err); } From ec7caea36874f4c27c683e8b2c6c4978054a7911 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sun, 15 Mar 2015 01:45:24 -0400 Subject: [PATCH 6/8] fix search on mongo --- src/database/mongo/main.js | 16 ++++++++++++---- src/search.js | 6 +----- src/user.js | 10 +++++++++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js index 913696aa9c..469a9720d9 100644 --- a/src/database/mongo/main.js +++ b/src/database/mongo/main.js @@ -34,12 +34,20 @@ module.exports = function(db, module) { searchQuery.$text = {$search: data.content}; } - if (data.cid) { - searchQuery.cid = data.cid; + if (Array.isArray(data.cid) && data.cid.length) { + if (data.cid.length > 1) { + searchQuery.cid = {$in: data.cid.map(String)}; + } else { + searchQuery.cid = data.cid[0].toString(); + } } - if (data.uid) { - searchQuery.uid = data.uid; + if (Array.isArray(data.uid) && data.uid.length) { + if (data.uid.length > 1) { + searchQuery.uid = {$in: data.uid.map(String)}; + } else { + searchQuery.uid = data.uid[0].toString(); + } } db.collection('search').find(searchQuery, {limit: limit}).toArray(function(err, results) { diff --git a/src/search.js b/src/search.js index 257ce6f352..063ebfa142 100644 --- a/src/search.js +++ b/src/search.js @@ -414,11 +414,7 @@ function getChildrenCids(cids, uid, callback) { function getSearchUids(data, callback) { if (data.postedBy) { - if (Array.isArray(data.postedBy)) { - user.getUidsByUsernames(data.postedBy, callback); - } else { - user.getUidByUsername([data.postedBy], callback); - } + user.getUidsByUsernames(Array.isArray(data.postedBy) ? data.postedBy : [data.postedBy], callback); } else { callback(null, []); } diff --git a/src/user.js b/src/user.js index bd0eed4ef4..cb4492765a 100644 --- a/src/user.js +++ b/src/user.js @@ -335,7 +335,15 @@ var async = require('async'), }; User.getUidsByUsernames = function(usernames, callback) { - db.getObjectFields('username:uid', usernames, callback); + db.getObjectFields('username:uid', usernames, function(err, users) { + if (err) { + return callback(err); + } + var uids = usernames.map(function(username) { + return users[username]; + }); + callback(null, uids); + }); }; User.getUidByUserslug = function(userslug, callback) { From 518aec6e76180078269bb17ffb56af880d58f9bd Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sun, 15 Mar 2015 03:18:24 -0400 Subject: [PATCH 7/8] modified indices, removed key 2 collections now --- src/database/mongo.js | 12 ++++++++---- src/database/mongo/main.js | 13 +++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 9e4399b031..f97eeb6651 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -110,10 +110,14 @@ createIndex('objects', {_key: 1, value: -1}, {background:true}); createIndex('objects', {expireAt: 1}, {expireAfterSeconds:0, background:true}); - createIndex('search', {content:'text'}, {background:true}); - createIndex('search', {key: 1, id: 1}, {background:true}); - createIndex('search', {cid: 1}, {background:true}); - createIndex('search', {uid: 1}, {background:true}); + + createIndex('searchtopic', {content: 'text', uid: 1, cid: 1}, {background:true}); + createIndex('searchtopic', {id: 1}, {background:true}); + + + createIndex('searchpost', {content: 'text', uid: 1, cid: 1}, {background:true}); + createIndex('searchpost', {id: 1}, {background:true}); + if (typeof callback === 'function') { callback(); diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js index 469a9720d9..5e027843e0 100644 --- a/src/database/mongo/main.js +++ b/src/database/mongo/main.js @@ -8,8 +8,7 @@ module.exports = function(db, module) { module.searchIndex = function(key, data, id, callback) { callback = callback || function() {}; var setData = { - id: id, - key: key + id: id }; for(var field in data) { if (data.hasOwnProperty(field) && data[field]) { @@ -17,7 +16,7 @@ module.exports = function(db, module) { } } - db.collection('search').update({key: key, id: id}, {$set: setData}, {upsert:true, w: 1}, function(err) { + db.collection('search' + key).update({id: id}, {$set: setData}, {upsert:true, w: 1}, function(err) { if(err) { winston.error('Error indexing ' + err.message); } @@ -26,9 +25,7 @@ module.exports = function(db, module) { }; module.search = function(key, data, limit, callback) { - var searchQuery = { - key: key - }; + var searchQuery = {}; if (data.content) { searchQuery.$text = {$search: data.content}; @@ -50,7 +47,7 @@ module.exports = function(db, module) { } } - db.collection('search').find(searchQuery, {limit: limit}).toArray(function(err, results) { + db.collection('search' + key).find(searchQuery, {limit: limit}).toArray(function(err, results) { if (err) { return callback(err); } @@ -72,7 +69,7 @@ module.exports = function(db, module) { if (!id) { return callback(); } - db.collection('search').remove({key: key, id: id}, function(err, res) { + db.collection('search' + key).remove({id: id}, function(err, res) { callback(err); }); }; From 0cc900bdcfc20db4de8ba29718ba8bfea5135b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Thu, 19 Mar 2015 12:32:05 -0400 Subject: [PATCH 8/8] closes #2878 --- public/src/client/category.js | 4 ++-- public/src/client/topic.js | 14 +++++++------- public/src/modules/navigator.js | 18 +++++++++++++----- public/src/modules/share.js | 2 +- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/public/src/client/category.js b/public/src/client/category.js index 2a6a580323..bf91059a25 100644 --- a/public/src/client/category.js +++ b/public/src/client/category.js @@ -89,8 +89,8 @@ define('forum/category', [ }); }; - Category.navigatorCallback = function(element, elementCount) { - return parseInt(element.attr('data-index'), 10) + 1; + Category.navigatorCallback = function(topIndex, bottomIndex, elementCount) { + return bottomIndex; }; $(window).on('action:popstate', function(ev, data) { diff --git a/public/src/client/topic.js b/public/src/client/topic.js index 4a597b4853..cbf4074d4f 100644 --- a/public/src/client/topic.js +++ b/public/src/client/topic.js @@ -148,18 +148,18 @@ define('forum/topic', [ return index; }; - Topic.navigatorCallback = function(element, elementCount) { + Topic.navigatorCallback = function(topPostIndex, bottomPostIndex, elementCount) { var path = ajaxify.removeRelativePath(window.location.pathname.slice(1)); if (!path.startsWith('topic')) { return 1; } - var postIndex = parseInt(element.attr('data-index'), 10); - var index = postIndex + 1; + var postIndex = topPostIndex; + var index = bottomPostIndex; if (config.topicPostSort !== 'oldest_to_newest') { - if (postIndex === 0) { + if (bottomPostIndex === 0) { index = 1; } else { - index = Math.max(elementCount - postIndex + 1, 1); + index = Math.max(elementCount - bottomPostIndex + 2, 1); } } @@ -175,8 +175,8 @@ define('forum/topic', [ var topicId = parts[1], slug = parts[2]; var newUrl = 'topic/' + topicId + '/' + (slug ? slug : ''); - if (postIndex > 0) { - newUrl += '/' + (postIndex + 1); + if (postIndex > 1) { + newUrl += '/' + postIndex; } if (newUrl !== currentUrl) { diff --git a/public/src/modules/navigator.js b/public/src/modules/navigator.js index 49b7a8f7ca..697046bd28 100644 --- a/public/src/modules/navigator.js +++ b/public/src/modules/navigator.js @@ -90,18 +90,26 @@ define('navigator', ['forum/pagination'], function(pagination) { navigator.update = function() { toggle(!!count); - $($(navigator.selector).get().reverse()).each(function() { + var topIndex = 0; + var bottomIndex = 0; + $(navigator.selector).each(function() { var el = $(this); if (elementInView(el)) { - if (typeof navigator.callback === 'function') { - index = navigator.callback(el, count); - navigator.updateTextAndProgressBar(); + if (!topIndex) { + topIndex = parseInt(el.attr('data-index'), 10) + 1; + } else { + bottomIndex = parseInt(el.attr('data-index'), 10) + 1; } - + } else if (topIndex && bottomIndex) { return false; } }); + + if (typeof navigator.callback === 'function' && topIndex && bottomIndex) { + index = navigator.callback(topIndex, bottomIndex, count); + navigator.updateTextAndProgressBar(); + } }; navigator.updateTextAndProgressBar = function() { diff --git a/public/src/modules/share.js b/public/src/modules/share.js index 83c2dce041..6665a30b8c 100644 --- a/public/src/modules/share.js +++ b/public/src/modules/share.js @@ -50,7 +50,7 @@ define('share', function() { function getPostUrl(clickedElement) { var parts = window.location.pathname.split('/'); - var postIndex = parseInt(clickedElement.parents('data-index').attr('data-index'), 10); + var postIndex = parseInt(clickedElement.parents('[data-index]').attr('data-index'), 10); return '/' + parts[1] + '/' + parts[2] + (parts[3] ? '/' + parts[3] : '') + (postIndex ? '/' + (postIndex + 1) : ''); }