hyperzlib
/
nodebb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

part 1 of search change

Browse Source
v1.18.x
barisusakli 10 years ago
parent fd5f00459b
commit c7128bcc98
6 changed files with 125 additions and 96 deletions
Show Stats Download Patch File Download Diff File

2
src/database/mongo.js
Unescape Escape View File

@ -112,6 +112,8 @@
createIndex('search', {content:'text'}, {background:true});
createIndex('search', {key: 1, id: 1}, {background:true});
createIndex('search', {cid: 1}, {background:true});
createIndex('search', {uid: 1}, {background:true});
if (typeof callback === 'function') {
callback();

38
src/database/mongo/main.js
Unescape Escape View File

@ -5,15 +5,19 @@ var winston = require('winston');
module.exports = function(db, module) {
var helpers = module.helpers.mongo;
module.searchIndex = function(key, content, id, callback) {
module.searchIndex = function(key, data, id, callback) {
callback = callback || function() {};
var data = {
var setData = {
id: id,
key: key,
content: content
key: key
};
for(var field in data) {
if (data.hasOwnProperty(field)) {
setData[field] = data[field];
}
}
db.collection('search').update({key:key, id:id}, {$set:data}, {upsert:true, w: 1}, function(err) {
db.collection('search').update({key: key, id: id}, {$set: setData}, {upsert:true, w: 1}, function(err) {
if(err) {
winston.error('Error indexing ' + err.message);
}
@ -21,13 +25,29 @@ module.exports = function(db, module) {
});
};
module.search = function(key, term, limit, callback) {
db.collection('search').find({ $text: { $search: term }, key: key}, {limit: limit}).toArray(function(err, results) {
if(err) {
module.search = function(key, data, limit, callback) {
var searchQuery = {
key: key
};
if (data.content) {
searchQuery.$text = {$search: data.content};
}
if (data.cid) {
searchQuery.cid = data.cid;
}
if (data.uid) {
searchQuery.uid = data.uid;
}
db.collection('search').find(searchQuery, {limit: limit}).toArray(function(err, results) {
if (err) {
return callback(err);
}
if(!results || !results.length) {
if (!results || !results.length) {
return callback(null, []);
}

12
src/database/redis.js
Unescape Escape View File

@ -9,7 +9,7 @@
utils = require('./../../public/src/utils.js'),
redis,
connectRedis,
reds,
redisSearch,
redisClient,
postSearch,
topicSearch;
@ -41,7 +41,7 @@
try {
redis = require('redis');
connectRedis = require('connect-redis')(session);
reds = require('reds');
redisSearch = require('redisearch');
} catch (err) {
winston.error('Unable to initialize Redis! Is Redis installed? Error :' + err.message);
process.exit();
@ -56,12 +56,8 @@
ttl: 60 * 60 * 24 * 14
});
reds.createClient = function () {
return reds.client || (reds.client = redisClient);
};
module.postSearch = reds.createSearch('nodebbpostsearch');
module.topicSearch = reds.createSearch('nodebbtopicsearch');
module.postSearch = redisSearch.createSearch('nodebbpostsearch', redisClient);
module.topicSearch = redisSearch.createSearch('nodebbtopicsearch', redisClient);
require('./redis/main')(redisClient, module);
require('./redis/hash')(redisClient, module);

22
src/database/redis/main.js
Unescape Escape View File

@ -1,27 +1,19 @@
"use strict";
module.exports = function(redisClient, module) {
module.searchIndex = function(key, content, id, callback) {
module.searchIndex = function(key, data, id, callback) {
if (key === 'post') {
module.postSearch.index(content, id, callback);
module.postSearch.index(data, id, callback);
} else if(key === 'topic') {
module.topicSearch.index(content, id, callback);
module.topicSearch.index(data, id, callback);
}
};
module.search = function(key, term, limit, callback) {
function search(searchObj, callback) {
searchObj
.query(term)
.between(0, limit - 1)
.type('or')
.end(callback);
}
if(key === 'post') {
search(module.postSearch, callback);
module.search = function(key, data, limit, callback) {
if (key === 'post') {
module.postSearch.query(data, 0, limit - 1, callback);
} else if(key === 'topic') {
search(module.topicSearch, callback);
module.topicSearch.query(data, 0, limit - 1, callback);
}
};

141
src/search.js
Unescape Escape View File

@ -43,7 +43,7 @@ search.search = function(data, callback) {
};
if (searchIn === 'posts' || searchIn === 'titles' || searchIn === 'titlesposts') {
searchInContent(query, data, done);
searchInContent(data, done);
} else if (searchIn === 'users') {
searchInUsers(query, data.uid, done);
} else if (searchIn === 'tags') {
@ -53,77 +53,80 @@ search.search = function(data, callback) {
}
};
function searchInContent(query, data, callback) {
function searchInContent(data, callback) {
data.uid = data.uid || 0;
async.parallel({
pids: function(next) {
if (data.searchIn === 'posts' || data.searchIn === 'titlesposts') {
search.searchQuery('post', query, next);
} else {
next(null, []);
}
},
tids: function(next) {
if (data.searchIn === 'titles' || data.searchIn === 'titlesposts') {
search.searchQuery('topic', query, next);
} else {
next(null, []);
}
searchCids: function(next) {
getSearchCids(data, next);
},
searchCategories: function(next) {
getSearchCategories(data, next);
searchUids: function(next) {
getSearchUids(data, next);
}
}, function (err, results) {
}, function(err, results) {
if (err) {
return callback(err);
}
var matchCount = 0;
if (!results || (!results.pids.length && !results.tids.length)) {
return callback(null, {matches: [], matchCount: matchCount});
}
async.waterfall([
function(next) {
topics.getMainPids(results.tids, next);
},
function(mainPids, next) {
results.pids = mainPids.concat(results.pids).filter(function(pid, index, array) {
return pid && array.indexOf(pid) === index;
});
privileges.posts.filter('read', results.pids, data.uid, next);
},
function(pids, next) {
filterAndSort(pids, data, results.searchCategories, next);
},
function(pids, next) {
matchCount = pids.length;
if (data.page) {
var start = Math.max(0, (data.page - 1)) * 10;
pids = pids.slice(start, start + 10);
async.parallel({
pids: function(next) {
if (data.searchIn === 'posts' || data.searchIn === 'titlesposts') {
search.searchQuery('post', data.query, results.searchCids, results.searchUids, next);
} else {
next(null, []);
}
posts.getPostSummaryByPids(pids, data.uid, {stripTags: true, parse: false}, next);
},
function(posts, next) {
next(null, {matches: posts, matchCount: matchCount});
tids: function(next) {
if (data.searchIn === 'titles' || data.searchIn === 'titlesposts') {
search.searchQuery('topic', data.query, results.searchCids, results.searchUids, next);
} else {
next(null, []);
}
}
}, function (err, results) {
if (err) {
return callback(err);
}
console.log(results)
var matchCount = 0;
if (!results || (!results.pids.length && !results.tids.length)) {
return callback(null, {matches: [], matchCount: matchCount});
}
], callback);
async.waterfall([
function(next) {
topics.getMainPids(results.tids, next);
},
function(mainPids, next) {
results.pids = mainPids.concat(results.pids).filter(function(pid, index, array) {
return pid && array.indexOf(pid) === index;
});
privileges.posts.filter('read', results.pids, data.uid, next);
},
function(pids, next) {
filterAndSort(pids, data, results.searchCategories, next);
},
function(pids, next) {
matchCount = pids.length;
if (data.page) {
var start = Math.max(0, (data.page - 1)) * 10;
pids = pids.slice(start, start + 10);
}
posts.getPostSummaryByPids(pids, data.uid, {stripTags: true, parse: false}, next);
},
function(posts, next) {
next(null, {matches: posts, matchCount: matchCount});
}
], callback);
});
});
}
function filterAndSort(pids, data, searchCategories, callback) {
function filterAndSort(pids, data, callback) {
async.parallel({
posts: function(next) {
getMatchedPosts(pids, data, searchCategories, next);
},
postedByUid: function(next) {
if (data.postedBy) {
user.getUidByUsername(data.postedBy, next);
} else {
next();
}
getMatchedPosts(pids, data, next);
}
}, function(err, results) {
if (err) {
@ -134,8 +137,6 @@ function filterAndSort(pids, data, searchCategories, callback) {
}
var posts = results.posts.filter(Boolean);
posts = filterByUser(posts, results.postedByUid);
posts = filterByCategories(posts, searchCategories);
posts = filterByPostcount(posts, data.replies, data.repliesFilter);
posts = filterByTimerange(posts, data.timeRange, data.timeFilter);
@ -149,7 +150,7 @@ function filterAndSort(pids, data, searchCategories, callback) {
});
}
function getMatchedPosts(pids, data, searchCategories, callback) {
function getMatchedPosts(pids, data, callback) {
var postFields = ['pid', 'tid', 'timestamp'];
var topicFields = [];
var categoryFields = [];
@ -158,7 +159,7 @@ function getMatchedPosts(pids, data, searchCategories, callback) {
postFields.push('uid');
}
if (searchCategories.length || (data.sortBy && data.sortBy.startsWith('category.'))) {
if (data.sortBy && data.sortBy.startsWith('category.')) {
topicFields.push('cid');
}
@ -389,7 +390,7 @@ function sortPosts(posts, data) {
}
}
function getSearchCategories(data, callback) {
function getSearchCids(data, callback) {
if (!Array.isArray(data.categories) || !data.categories.length || data.categories.indexOf('all') !== -1) {
return callback(null, []);
}
@ -439,6 +440,18 @@ function getChildrenCids(cids, uid, callback) {
});
}
function getSearchUids(data, callback) {
if (data.postedBy) {
if (Array.isArray(data.postedBy)) {
user.getUidsByUsernames(data.postedBy, callback);
} else {
user.getUidByUsername([data.postedBy], callback);
}
} else {
callback(null, []);
}
}
function searchInUsers(query, uid, callback) {
user.search({query: query, uid: uid}, function(err, results) {
if (err) {
@ -458,10 +471,12 @@ function searchInTags(query, callback) {
});
}
search.searchQuery = function(index, query, callback) {
search.searchQuery = function(index, content, cids, uids, callback) {
plugins.fireHook('filter:search.query', {
index: index,
query: query
content: content,
cids: cids,
uids: uids
}, callback);
};

6
src/user.js
Unescape Escape View File

@ -158,7 +158,7 @@ var async = require('async'),
if (now - parseInt(userOnlineTime, 10) < 300000) {
return callback();
}
db.sortedSetAdd('users:online', now, uid, next);
db.sortedSetAdd('users:online', now, uid, next);
},
function(next) {
topics.pushUnreadCount(uid);
@ -334,6 +334,10 @@ var async = require('async'),
db.getObjectField('username:uid', username, callback);
};
User.getUidsByUsernames = function(usernames, callback) {
db.getObjectFields('username:uid', usernames, callback);
};
User.getUidByUserslug = function(userslug, callback) {
if (!userslug) {
return callback();

Write Preview
Loading…
Cancel
Save