From c7128bcc989bac77bb810f3aaa2964892b59dd53 Mon Sep 17 00:00:00 2001 From: barisusakli Date: Sat, 14 Mar 2015 23:00:24 -0400 Subject: [PATCH] part 1 of search change --- src/database/mongo.js | 2 + src/database/mongo/main.js | 38 +++++++--- src/database/redis.js | 12 ++-- src/database/redis/main.js | 22 ++---- src/search.js | 141 ++++++++++++++++++++----------------- src/user.js | 6 +- 6 files changed, 125 insertions(+), 96 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 0e69c12380..9e4399b031 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -112,6 +112,8 @@ createIndex('search', {content:'text'}, {background:true}); createIndex('search', {key: 1, id: 1}, {background:true}); + createIndex('search', {cid: 1}, {background:true}); + createIndex('search', {uid: 1}, {background:true}); if (typeof callback === 'function') { callback(); diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js index 029b8f1fee..a40567e2a6 100644 --- a/src/database/mongo/main.js +++ b/src/database/mongo/main.js @@ -5,15 +5,19 @@ var winston = require('winston'); module.exports = function(db, module) { var helpers = module.helpers.mongo; - module.searchIndex = function(key, content, id, callback) { + module.searchIndex = function(key, data, id, callback) { callback = callback || function() {}; - var data = { + var setData = { id: id, - key: key, - content: content + key: key }; + for(var field in data) { + if (data.hasOwnProperty(field)) { + setData[field] = data[field]; + } + } - db.collection('search').update({key:key, id:id}, {$set:data}, {upsert:true, w: 1}, function(err) { + db.collection('search').update({key: key, id: id}, {$set: setData}, {upsert:true, w: 1}, function(err) { if(err) { winston.error('Error indexing ' + err.message); } @@ -21,13 +25,29 @@ module.exports = function(db, module) { }); }; - module.search = function(key, term, limit, callback) { - db.collection('search').find({ $text: { $search: term }, key: key}, {limit: limit}).toArray(function(err, results) { - if(err) { + module.search = function(key, data, limit, callback) { + var searchQuery = { + key: key + }; + + if (data.content) { + searchQuery.$text = {$search: data.content}; + } + + if (data.cid) { + searchQuery.cid = data.cid; + } + + if (data.uid) { + searchQuery.uid = data.uid; + } + + db.collection('search').find(searchQuery, {limit: limit}).toArray(function(err, results) { + if (err) { return callback(err); } - if(!results || !results.length) { + if (!results || !results.length) { return callback(null, []); } diff --git a/src/database/redis.js b/src/database/redis.js index 727705c130..716a8fd715 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -9,7 +9,7 @@ utils = require('./../../public/src/utils.js'), redis, connectRedis, - reds, + redisSearch, redisClient, postSearch, topicSearch; @@ -41,7 +41,7 @@ try { redis = require('redis'); connectRedis = require('connect-redis')(session); - reds = require('reds'); + redisSearch = require('redisearch'); } catch (err) { winston.error('Unable to initialize Redis! Is Redis installed? Error :' + err.message); process.exit(); @@ -56,12 +56,8 @@ ttl: 60 * 60 * 24 * 14 }); - reds.createClient = function () { - return reds.client || (reds.client = redisClient); - }; - - module.postSearch = reds.createSearch('nodebbpostsearch'); - module.topicSearch = reds.createSearch('nodebbtopicsearch'); + module.postSearch = redisSearch.createSearch('nodebbpostsearch', redisClient); + module.topicSearch = redisSearch.createSearch('nodebbtopicsearch', redisClient); require('./redis/main')(redisClient, module); require('./redis/hash')(redisClient, module); diff --git a/src/database/redis/main.js b/src/database/redis/main.js index 094fa3b695..9ed956f0b3 100644 --- a/src/database/redis/main.js +++ b/src/database/redis/main.js @@ -1,27 +1,19 @@ "use strict"; module.exports = function(redisClient, module) { - module.searchIndex = function(key, content, id, callback) { + module.searchIndex = function(key, data, id, callback) { if (key === 'post') { - module.postSearch.index(content, id, callback); + module.postSearch.index(data, id, callback); } else if(key === 'topic') { - module.topicSearch.index(content, id, callback); + module.topicSearch.index(data, id, callback); } }; - module.search = function(key, term, limit, callback) { - function search(searchObj, callback) { - searchObj - .query(term) - .between(0, limit - 1) - .type('or') - .end(callback); - } - - if(key === 'post') { - search(module.postSearch, callback); + module.search = function(key, data, limit, callback) { + if (key === 'post') { + module.postSearch.query(data, 0, limit - 1, callback); } else if(key === 'topic') { - search(module.topicSearch, callback); + module.topicSearch.query(data, 0, limit - 1, callback); } }; diff --git a/src/search.js b/src/search.js index e44dedc216..a478d7f8e9 100644 --- a/src/search.js +++ b/src/search.js @@ -43,7 +43,7 @@ search.search = function(data, callback) { }; if (searchIn === 'posts' || searchIn === 'titles' || searchIn === 'titlesposts') { - searchInContent(query, data, done); + searchInContent(data, done); } else if (searchIn === 'users') { searchInUsers(query, data.uid, done); } else if (searchIn === 'tags') { @@ -53,77 +53,80 @@ search.search = function(data, callback) { } }; -function searchInContent(query, data, callback) { +function searchInContent(data, callback) { data.uid = data.uid || 0; async.parallel({ - pids: function(next) { - if (data.searchIn === 'posts' || data.searchIn === 'titlesposts') { - search.searchQuery('post', query, next); - } else { - next(null, []); - } - }, - tids: function(next) { - if (data.searchIn === 'titles' || data.searchIn === 'titlesposts') { - search.searchQuery('topic', query, next); - } else { - next(null, []); - } + searchCids: function(next) { + getSearchCids(data, next); }, - searchCategories: function(next) { - getSearchCategories(data, next); + searchUids: function(next) { + getSearchUids(data, next); } - }, function (err, results) { + }, function(err, results) { if (err) { return callback(err); } - var matchCount = 0; - if (!results || (!results.pids.length && !results.tids.length)) { - return callback(null, {matches: [], matchCount: matchCount}); - } - - async.waterfall([ - function(next) { - topics.getMainPids(results.tids, next); - }, - function(mainPids, next) { - results.pids = mainPids.concat(results.pids).filter(function(pid, index, array) { - return pid && array.indexOf(pid) === index; - }); - - privileges.posts.filter('read', results.pids, data.uid, next); - }, - function(pids, next) { - filterAndSort(pids, data, results.searchCategories, next); - }, - function(pids, next) { - matchCount = pids.length; - if (data.page) { - var start = Math.max(0, (data.page - 1)) * 10; - pids = pids.slice(start, start + 10); + async.parallel({ + pids: function(next) { + if (data.searchIn === 'posts' || data.searchIn === 'titlesposts') { + search.searchQuery('post', data.query, results.searchCids, results.searchUids, next); + } else { + next(null, []); } - - posts.getPostSummaryByPids(pids, data.uid, {stripTags: true, parse: false}, next); }, - function(posts, next) { - next(null, {matches: posts, matchCount: matchCount}); + tids: function(next) { + if (data.searchIn === 'titles' || data.searchIn === 'titlesposts') { + search.searchQuery('topic', data.query, results.searchCids, results.searchUids, next); + } else { + next(null, []); + } + } + }, function (err, results) { + if (err) { + return callback(err); + } +console.log(results) + var matchCount = 0; + if (!results || (!results.pids.length && !results.tids.length)) { + return callback(null, {matches: [], matchCount: matchCount}); } - ], callback); + + async.waterfall([ + function(next) { + topics.getMainPids(results.tids, next); + }, + function(mainPids, next) { + results.pids = mainPids.concat(results.pids).filter(function(pid, index, array) { + return pid && array.indexOf(pid) === index; + }); + + privileges.posts.filter('read', results.pids, data.uid, next); + }, + function(pids, next) { + filterAndSort(pids, data, results.searchCategories, next); + }, + function(pids, next) { + matchCount = pids.length; + if (data.page) { + var start = Math.max(0, (data.page - 1)) * 10; + pids = pids.slice(start, start + 10); + } + + posts.getPostSummaryByPids(pids, data.uid, {stripTags: true, parse: false}, next); + }, + function(posts, next) { + next(null, {matches: posts, matchCount: matchCount}); + } + ], callback); + }); }); } -function filterAndSort(pids, data, searchCategories, callback) { +function filterAndSort(pids, data, callback) { async.parallel({ posts: function(next) { - getMatchedPosts(pids, data, searchCategories, next); - }, - postedByUid: function(next) { - if (data.postedBy) { - user.getUidByUsername(data.postedBy, next); - } else { - next(); - } + getMatchedPosts(pids, data, next); } }, function(err, results) { if (err) { @@ -134,8 +137,6 @@ function filterAndSort(pids, data, searchCategories, callback) { } var posts = results.posts.filter(Boolean); - posts = filterByUser(posts, results.postedByUid); - posts = filterByCategories(posts, searchCategories); posts = filterByPostcount(posts, data.replies, data.repliesFilter); posts = filterByTimerange(posts, data.timeRange, data.timeFilter); @@ -149,7 +150,7 @@ function filterAndSort(pids, data, searchCategories, callback) { }); } -function getMatchedPosts(pids, data, searchCategories, callback) { +function getMatchedPosts(pids, data, callback) { var postFields = ['pid', 'tid', 'timestamp']; var topicFields = []; var categoryFields = []; @@ -158,7 +159,7 @@ function getMatchedPosts(pids, data, searchCategories, callback) { postFields.push('uid'); } - if (searchCategories.length || (data.sortBy && data.sortBy.startsWith('category.'))) { + if (data.sortBy && data.sortBy.startsWith('category.')) { topicFields.push('cid'); } @@ -389,7 +390,7 @@ function sortPosts(posts, data) { } } -function getSearchCategories(data, callback) { +function getSearchCids(data, callback) { if (!Array.isArray(data.categories) || !data.categories.length || data.categories.indexOf('all') !== -1) { return callback(null, []); } @@ -439,6 +440,18 @@ function getChildrenCids(cids, uid, callback) { }); } +function getSearchUids(data, callback) { + if (data.postedBy) { + if (Array.isArray(data.postedBy)) { + user.getUidsByUsernames(data.postedBy, callback); + } else { + user.getUidByUsername([data.postedBy], callback); + } + } else { + callback(null, []); + } +} + function searchInUsers(query, uid, callback) { user.search({query: query, uid: uid}, function(err, results) { if (err) { @@ -458,10 +471,12 @@ function searchInTags(query, callback) { }); } -search.searchQuery = function(index, query, callback) { +search.searchQuery = function(index, content, cids, uids, callback) { plugins.fireHook('filter:search.query', { index: index, - query: query + content: content, + cids: cids, + uids: uids }, callback); }; diff --git a/src/user.js b/src/user.js index 109e67a3f0..bd0eed4ef4 100644 --- a/src/user.js +++ b/src/user.js @@ -158,7 +158,7 @@ var async = require('async'), if (now - parseInt(userOnlineTime, 10) < 300000) { return callback(); } - db.sortedSetAdd('users:online', now, uid, next); + db.sortedSetAdd('users:online', now, uid, next); }, function(next) { topics.pushUnreadCount(uid); @@ -334,6 +334,10 @@ var async = require('async'), db.getObjectField('username:uid', username, callback); }; + User.getUidsByUsernames = function(usernames, callback) { + db.getObjectFields('username:uid', usernames, callback); + }; + User.getUidByUserslug = function(userslug, callback) { if (!userslug) { return callback();