From 53ca7a114369a99916ab9265d7a86e69559d293f Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 20:06:36 -0500 Subject: [PATCH] added search functions to database files, removed reds from nodebb moved it to redis --- src/database/mongo.js | 54 ++++++++++++++++++++++++++++++++++++++----- src/database/redis.js | 39 +++++++++++++++++++++++++++++++ src/postTools.js | 16 ++++++------- src/posts.js | 8 +++---- src/routes/api.js | 16 +++---------- src/routes/debug.js | 4 +++- src/threadTools.js | 7 ++---- src/topics.js | 4 +--- src/user.js | 9 ++++---- 9 files changed, 110 insertions(+), 47 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index d5b18b85dd..699c46af6f 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -39,6 +39,20 @@ } }); + db.createCollection('search', function(err, collection) { + if(err) { + winston.error("Error creating collection " + err.message); + return; + } + if(collection) { + collection.ensureIndex({content:'text'}, {background:true}, function(err, name){ + if(err) { + winston.error("Error creating index " + err.message); + } + }); + } + }) + callback(err); }); @@ -54,6 +68,28 @@ // Exported functions // + module.searchIndex = function(key, content, id) { + + db.collection('search').update({key:key, content:content, _id:id}, {upsert:true, w: 1}, function(err, result) { + + }); + } + + module.search = function(key, term, callback) { + db.command({text:"search" , search: term }, function(err, result) { + callback(err, result) + }); + /*db.runCommand("text", { search: term }, function(err, result) { + callback(err, result); + });*/ + } + + module.searchRemove = function(key, id) { + db.collection('search').remove({_id:id}, function(err, result) { + callback(err, result); + }); + } + module.flushdb = function(callback) { db.dropDatabase(function(err, result) { if(err){ @@ -72,14 +108,20 @@ module.info = function(callback) { db.stats({scale:1024}, function(err, stats) { - stats.avgObjSize = (stats.avgObjSize / 1024).toFixed(2); + db.serverStatus(function(err, serverStatus) { + + stats.avgObjSize = (stats.avgObjSize / 1024).toFixed(2); - stats.raw = JSON.stringify(stats, null, 4); + stats.serverStatus = serverStatus; - stats.mongo = true; - //remove this when andrew adds in undefined checking to templates - stats.redis = false; - callback(err, stats); + stats.raw = JSON.stringify(stats, null, 4); + + stats.mongo = true; + //remove this when andrew adds in undefined checking to templates + stats.redis = false; + callback(err, stats); + + }); }); } diff --git a/src/database/redis.js b/src/database/redis.js index 76fa920c84..7361be3458 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -33,6 +33,10 @@ return reds.client || (reds.client = redisClient); }; + var userSearch = reds.createSearch('nodebbusersearch'), + postSearch = reds.createSearch('nodebbpostsearch'), + topicSearch = reds.createSearch('nodebbtopicsearch'); + if (nconf.get('redis:password')) { redisClient.auth(nconf.get('redis:password')); } @@ -69,6 +73,41 @@ // // Exported functions // + module.searchIndex = function(key, content, id) { + if(key === 'post') { + postSearch.index(content, id); + } else if(key === 'topic') { + topicSearch.index(content, id); + } else if(key === 'user') { + userSearch.index(content, id); + } + } + + module.search = function(key, term, callback) { + function search(searchObj, callback) { + searchObj + .query(query = term).type('or') + .end(callback); + } + + if(key === 'post') { + search(postSearch, callback); + } else if(key === 'topic') { + search(topicSearch, callback); + } else if(key === 'user') { + search(userSearch, callback); + } + } + + module.searchRemove = function(key, id) { + if(key === 'post') { + postSearch.remove(id); + } else if(key === 'topic') { + topicSearch.remove(id); + } else if(key === 'user') { + userSearch.remove(id); + } + } module.flushdb = function(callback) { redisClient.send_command('flushdb', [], function(err) { diff --git a/src/postTools.js b/src/postTools.js index 0ec6abc81a..e31fa70552 100644 --- a/src/postTools.js +++ b/src/postTools.js @@ -10,9 +10,7 @@ var db = require('./database'), utils = require('../public/src/utils'), plugins = require('./plugins'), - reds = require('reds'), - postSearch = reds.createSearch('nodebbpostsearch'), - topicSearch = reds.createSearch('nodebbtopicsearch'), + winston = require('winston'), meta = require('./meta'), Feed = require('./feed'); @@ -83,8 +81,8 @@ var db = require('./database'), } ]); - postSearch.remove(pid, function() { - postSearch.index(content, pid); + db.searchRemove('post', pid, function() { + db.searchIndex('post', content, pid); }); async.parallel([ @@ -93,8 +91,8 @@ var db = require('./database'), PostTools.isMain(pid, tid, function(err, isMainPost) { if (isMainPost) { topics.setTopicField(tid, 'title', title); - topicSearch.remove(tid, function() { - topicSearch.index(title, tid); + db.searchRemove('topic', tid, function() { + db.searchIndex('topic', title, tid); }); } @@ -132,7 +130,7 @@ var db = require('./database'), var success = function() { posts.setPostField(pid, 'deleted', 1); db.decrObjectField('global', 'postCount'); - postSearch.remove(pid); + db.searchRemove('post', pid); posts.getPostFields(pid, ['tid', 'uid'], function(err, postData) { db.incrObjectFieldBy('topic:' + postData.tid, 'postcount', -1); @@ -203,7 +201,7 @@ var db = require('./database'), Feed.updateTopic(postData.tid); Feed.updateRecent(); - postSearch.index(postData.content, pid); + db.searchIndex('post', postData.content, pid); callback(); }); diff --git a/src/posts.js b/src/posts.js index 255707902a..1b72549454 100644 --- a/src/posts.js +++ b/src/posts.js @@ -12,8 +12,6 @@ var db = require('./database'), meta = require('./meta'), async = require('async'), - reds = require('reds'), - postSearch = reds.createSearch('nodebbpostsearch'), nconf = require('nconf'), validator = require('validator'), winston = require('winston'); @@ -108,7 +106,7 @@ var db = require('./database'), plugins.fireHook('action:post.save', postData); - postSearch.index(content, pid); + db.searchIndex('post', content, pid); callback(null, postData); }); @@ -481,10 +479,10 @@ var db = require('./database'), function reIndex(pid, callback) { Posts.getPostField(pid, 'content', function(err, content) { - postSearch.remove(pid, function() { + db.searchRemove('post', pid, function() { if (content && content.length) { - postSearch.index(content, pid); + db.searchIndex('post', content, pid); } callback(null); }); diff --git a/src/routes/api.js b/src/routes/api.js index b92a29e1f1..0bebd9f975 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -226,18 +226,8 @@ var path = require('path'), app.get('/search/:term', function (req, res, next) { - var reds = require('reds'); - var postSearch = reds.createSearch('nodebbpostsearch'); - var topicSearch = reds.createSearch('nodebbtopicsearch'); - - function search(searchObj, callback) { - searchObj - .query(query = req.params.term).type('or') - .end(callback); - } - function searchPosts(callback) { - search(postSearch, function (err, pids) { + db.search('post', req.params.term, function(err, pids) { if (err) { return callback(err, null); } @@ -248,11 +238,11 @@ var path = require('path'), } callback(null, posts); }); - }) + }); } function searchTopics(callback) { - search(topicSearch, function (err, tids) { + db.search('topic', req.params.term, function(err, tids) { if (err) { return callback(err, null); } diff --git a/src/routes/debug.js b/src/routes/debug.js index a7a3033b54..59838d83b1 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -407,9 +407,11 @@ var DebugRoute = function(app) { var miscTests = [ function(next) { - db.sortedSetScore('users:joindate', 1, function(err, result) { + //db.searchIndex('post', 'here is content tomato testing purple orange', 1); + db.search('post', 'tomato', function(err, result) { next(err, result); }) + } ]; diff --git a/src/threadTools.js b/src/threadTools.js index 939603958c..ed32f795be 100644 --- a/src/threadTools.js +++ b/src/threadTools.js @@ -8,9 +8,6 @@ var db = require('./database'), posts = require('./posts'), meta = require('./meta'), websockets = require('./websockets'); - - reds = require('reds'), - topicSearch = reds.createSearch('nodebbtopicsearch'), winston = require('winston'), nconf = require('nconf'), @@ -94,7 +91,7 @@ var db = require('./database'), ThreadTools.lock(tid); - topicSearch.remove(tid); + db.searchRemove('topic', tid); websockets.in('topic_' + tid).emit('event:topic_deleted', { tid: tid, @@ -117,7 +114,7 @@ var db = require('./database'), }); topics.getTopicField(tid, 'title', function(err, title) { - topicSearch.index(title, tid); + db.searchIndex('topic', title, tid); }); if(callback) { diff --git a/src/topics.js b/src/topics.js index df33bd8381..9d055556ad 100644 --- a/src/topics.js +++ b/src/topics.js @@ -2,8 +2,6 @@ var async = require('async'), gravatar = require('gravatar'), nconf = require('nconf'), validator = require('validator'), - reds = require('reds'), - topicSearch = reds.createSearch('nodebbtopicsearch'), db = require('./database'), posts = require('./posts'), @@ -84,7 +82,7 @@ var async = require('async'), 'pinned': 0 }); - topicSearch.index(title, tid); + db.searchIndex('topic', title, tid); user.addTopicIdToUser(uid, tid); diff --git a/src/user.js b/src/user.js index 8bc99bea91..c4459be153 100644 --- a/src/user.js +++ b/src/user.js @@ -4,7 +4,6 @@ var bcrypt = require('bcrypt'), nconf = require('nconf'), winston = require('winston'), gravatar = require('gravatar'), - userSearch = require('reds').createSearch('nodebbusersearch'), check = require('validator').check, sanitize = require('validator').sanitize, @@ -113,7 +112,7 @@ var bcrypt = require('bcrypt'), db.sortedSetAdd('users:postcount', 0, uid); db.sortedSetAdd('users:reputation', 0, uid); - userSearch.index(username, uid); + db.searchIndex('user', username, uid); if (password !== undefined) { User.hashPassword(password, function(err, hash) { @@ -390,8 +389,8 @@ var bcrypt = require('bcrypt'), } function reIndexUser(uid, username) { - userSearch.remove(uid, function() { - userSearch.index(username, uid); + db.searchRemove('user', uid, function() { + db.searchIndex('user', username, uid); }); } @@ -407,7 +406,7 @@ var bcrypt = require('bcrypt'), callback([]); return; } - userSearch.query(username).type('or').end(function(err, uids) { + db.search('user', username, (function(err, uids) { if (err) { console.log(err); return;