From 95efb2ae5e23bde3b0f4c3ef81f1b93883f165c8 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Mon, 2 Dec 2013 15:45:15 -0500 Subject: [PATCH 01/83] started dbal --- src/database.js | 6 ++++ src/database/mongo.js | 0 src/database/redis.js | 5 +++ src/redis.js | 78 ------------------------------------------- 4 files changed, 11 insertions(+), 78 deletions(-) create mode 100644 src/database.js create mode 100644 src/database/mongo.js create mode 100644 src/database/redis.js delete mode 100644 src/redis.js diff --git a/src/database.js b/src/database.js new file mode 100644 index 0000000000..7a9fdf943d --- /dev/null +++ b/src/database.js @@ -0,0 +1,6 @@ + + +var nconf = require('nconf'); + db = require('./databases/' + nconf.get('database')); + +module.exports = db; \ No newline at end of file diff --git a/src/database/mongo.js b/src/database/mongo.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/database/redis.js b/src/database/redis.js new file mode 100644 index 0000000000..9eb3347163 --- /dev/null +++ b/src/database/redis.js @@ -0,0 +1,5 @@ +var RDB = require('redis'); + + + +(function() \ No newline at end of file diff --git a/src/redis.js b/src/redis.js deleted file mode 100644 index d55bd57dd5..0000000000 --- a/src/redis.js +++ /dev/null @@ -1,78 +0,0 @@ -(function(module) { - 'use strict'; - - var RedisDB, - redis = require('redis'), - utils = require('./../public/src/utils.js'), - winston = require('winston'), - nconf = require('nconf'), - redis_socket_or_host = nconf.get('redis:host'); - - if (redis_socket_or_host && redis_socket_or_host.indexOf('/')>=0) { - /* If redis.host contains a path name character, use the unix dom sock connection. ie, /tmp/redis.sock */ - RedisDB = redis.createClient(nconf.get('redis:host')); - } else { - /* Else, connect over tcp/ip */ - RedisDB = redis.createClient(nconf.get('redis:port'), nconf.get('redis:host')); - } - - if (nconf.get('redis:password')) { - RedisDB.auth(nconf.get('redis:password')); - } - - var db = parseInt(nconf.get('redis:database'), 10); - if (db){ - RedisDB.select(db, function(error){ - if(error !== null){ - winston.error("NodeBB could not connect to your Redis database. Redis returned the following error: " + error.message); - process.exit(); - } - }); - } - - RedisDB.handle = function(error) { - if (error !== null) { - winston.err(error); - if (global.env !== 'production') { - throw new Error(error); - } - } - }; - - - /* - * A possibly more efficient way of doing multiple sismember calls - */ - RedisDB.sismembers = function(key, needles, callback) { - var tempkey = key + ':temp:' + utils.generateUUID(); - RedisDB.sadd(tempkey, needles, function() { - RedisDB.sinter(key, tempkey, function(err, data) { - RedisDB.del(tempkey); - callback(err, data); - }); - }); - }; - - /* - * gets fields of a hash as an object instead of an array - */ - RedisDB.hmgetObject = function(key, fields, callback) { - RedisDB.hmget(key, fields, function(err, data) { - - if(err) { - return callback(err, null); - } - - var returnData = {}; - - for (var i = 0, ii = fields.length; i < ii; ++i) { - returnData[fields[i]] = data[i]; - } - - callback(null, returnData); - }); - }; - - module.exports = RedisDB; - -}(module)); \ No newline at end of file From 80e7fd93c6207d0a5a92f844fdd9c8b8a5eeec55 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Mon, 2 Dec 2013 15:46:25 -0500 Subject: [PATCH 02/83] added redis --- src/database/redis.js | 81 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/src/database/redis.js b/src/database/redis.js index 9eb3347163..212a2d5fa7 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -1,5 +1,82 @@ -var RDB = require('redis'); +(function(module) { + 'use strict'; + var RedisDB, + redis = require('redis'), + winston = require('winston'), + nconf = require('nconf'), + redis_socket_or_host = nconf.get('redis:host'), + utils = require('./../public/src/utils.js'); + + if (redis_socket_or_host && redis_socket_or_host.indexOf('/')>=0) { + /* If redis.host contains a path name character, use the unix dom sock connection. ie, /tmp/redis.sock */ + RedisDB = redis.createClient(nconf.get('redis:host')); + } else { + /* Else, connect over tcp/ip */ + RedisDB = redis.createClient(nconf.get('redis:port'), nconf.get('redis:host')); + } + + if (nconf.get('redis:password')) { + RedisDB.auth(nconf.get('redis:password')); + } + + var db = parseInt(nconf.get('redis:database'), 10); + + if (db){ + RedisDB.select(db, function(error){ + if(error !== null){ + winston.error("NodeBB could not connect to your Redis database. Redis returned the following error: " + error.message); + process.exit(); + } + }); + } + + RedisDB.handle = function(error) { + if (error !== null) { + winston.err(error); + if (global.env !== 'production') { + throw new Error(error); + } + } + }; + + + /* + * A possibly more efficient way of doing multiple sismember calls + */ + RedisDB.sismembers = function(key, needles, callback) { + var tempkey = key + ':temp:' + utils.generateUUID(); + RedisDB.sadd(tempkey, needles, function() { + RedisDB.sinter(key, tempkey, function(err, data) { + RedisDB.del(tempkey); + callback(err, data); + }); + }); + }; + + /* + * gets fields of a hash as an object instead of an array + */ + RedisDB.hmgetObject = function(key, fields, callback) { + RedisDB.hmget(key, fields, function(err, data) { + + if(err) { + return callback(err, null); + } + + var returnData = {}; + + for (var i = 0, ii = fields.length; i < ii; ++i) { + returnData[fields[i]] = data[i]; + } + + callback(null, returnData); + }); + }; + + module.exports = RedisDB; + + +}(exports)); -(function() \ No newline at end of file From 4f654fb4896fc437bf48fc1ad165ce23fbed99c5 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Mon, 2 Dec 2013 16:19:30 -0500 Subject: [PATCH 03/83] more work --- src/database.js | 2 +- src/database/mongo.js | 72 ++++++++++++++++++++++++++++++++++++++++ src/database/redis.js | 76 +++++++++++++++++++++++++++---------------- src/meta.js | 76 +++++++++++++++++++------------------------ src/plugins.js | 3 +- 5 files changed, 157 insertions(+), 72 deletions(-) diff --git a/src/database.js b/src/database.js index 7a9fdf943d..465cd3bcd5 100644 --- a/src/database.js +++ b/src/database.js @@ -1,6 +1,6 @@ var nconf = require('nconf'); - db = require('./databases/' + nconf.get('database')); + db = require('./database/' + nconf.get('database')); module.exports = db; \ No newline at end of file diff --git a/src/database/mongo.js b/src/database/mongo.js index e69de29bb2..e9f495264b 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -0,0 +1,72 @@ + + +(function(module) { + 'use strict'; + var mongoClient, + mongo = require('mongo') + winston = require('winston'), + nconf = require('nconf'), + mongoHost = nconf.get('mongo:host'), + utils = require('./../../public/src/utils.js'); + + // temp, look this up + mongoClient = mongo.createClient(nconf.get('mongo:port'), nconf.get('mongo:host')); + + // look up how its done in mongo + /*if (nconf.get('mongo:password')) { + redisClient.auth(nconf.get('mongo:password')); + } + + var db = parseInt(nconf.get('mongo:database'), 10); + + if (db){ + mongoClient.select(db, function(error) { + if(error) { + winston.error("NodeBB could not connect to your Redis database. Redis returned the following error: " + error.message); + process.exit(); + } + }); + }*/ + + // + // Exported functions + // + module.getFileName = function(callback) { + // TODO : get mongodb filename + } + + + module.setObject = function(key, data, callback) { + // TODO : implement in mongo + } + + module.setObjectField = function(key, field, callback) { + // TODO : implement in mongo + } + + module.getObject = function(key, callback) { + // TODO : implement in mongo + } + + module.getObjectField = function(key, field, callback) { + // TODO : implement in mongo + } + + module.getObjectFields = function(key, fields, callback) { + // TODO : implement in mongo + } + + module.deleteObjectField = function(key, field, callback) { + // TODO : implement in mongo + } + + module.incrObjectField = function(key, field, value, callback) { + // TODO : implement in mongo + } + + + + + +}(exports)); + diff --git a/src/database/redis.js b/src/database/redis.js index 212a2d5fa7..bd6597f3c5 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -2,65 +2,76 @@ (function(module) { 'use strict'; - var RedisDB, + var redisClient, redis = require('redis'), winston = require('winston'), nconf = require('nconf'), redis_socket_or_host = nconf.get('redis:host'), - utils = require('./../public/src/utils.js'); + utils = require('./../../public/src/utils.js'); if (redis_socket_or_host && redis_socket_or_host.indexOf('/')>=0) { /* If redis.host contains a path name character, use the unix dom sock connection. ie, /tmp/redis.sock */ - RedisDB = redis.createClient(nconf.get('redis:host')); + redisClient = redis.createClient(nconf.get('redis:host')); } else { /* Else, connect over tcp/ip */ - RedisDB = redis.createClient(nconf.get('redis:port'), nconf.get('redis:host')); + redisClient = redis.createClient(nconf.get('redis:port'), nconf.get('redis:host')); } if (nconf.get('redis:password')) { - RedisDB.auth(nconf.get('redis:password')); + redisClient.auth(nconf.get('redis:password')); } var db = parseInt(nconf.get('redis:database'), 10); if (db){ - RedisDB.select(db, function(error){ - if(error !== null){ + redisClient.select(db, function(error) { + if(error) { winston.error("NodeBB could not connect to your Redis database. Redis returned the following error: " + error.message); process.exit(); } }); } - RedisDB.handle = function(error) { - if (error !== null) { - winston.err(error); - if (global.env !== 'production') { - throw new Error(error); - } - } - }; - - /* * A possibly more efficient way of doing multiple sismember calls */ - RedisDB.sismembers = function(key, needles, callback) { + function sismembers(key, needles, callback) { var tempkey = key + ':temp:' + utils.generateUUID(); - RedisDB.sadd(tempkey, needles, function() { - RedisDB.sinter(key, tempkey, function(err, data) { - RedisDB.del(tempkey); + redisClient.sadd(tempkey, needles, function() { + redisClient.sinter(key, tempkey, function(err, data) { + redisClient.del(tempkey); callback(err, data); }); }); }; - /* - * gets fields of a hash as an object instead of an array - */ - RedisDB.hmgetObject = function(key, fields, callback) { - RedisDB.hmget(key, fields, function(err, data) { + // + // Exported functions + // + module.setObject = function(key, data, callback) { + redisClient.hmset(key, data, callback); + } + module.setObjectField = function(key, field, callback) { + redisClient.hset(key, field, callback) + } + + module.getObject = function(key, callback) { + redisClient.hgetall(key, callback) + } + + module.getObjectField = function(key, field, callback) { + module.getObjectFields(key, [field], function(err, data) { + if(err) { + return callback(err); + } + + callback(null, data[field]); + }); + } + + module.getObjectFields = function(key, fields, callback) { + redisClient.hmget(key, fields, function(err, data) { if(err) { return callback(err, null); } @@ -73,9 +84,18 @@ callback(null, returnData); }); - }; + } + + module.deleteObjectField = function(key, field, callback) { + redisClient.hdel(key, field, callback); + } + + module.incrObjectField = function(key, field, value, callback) { + redisClient.hincrby(key, field, value, callback); + } + + - module.exports = RedisDB; }(exports)); diff --git a/src/meta.js b/src/meta.js index 792d213a84..8930c944ba 100644 --- a/src/meta.js +++ b/src/meta.js @@ -1,11 +1,13 @@ -var utils = require('./../public/src/utils.js'), - RDB = require('./redis.js'), - plugins = require('./plugins'), - async = require('async'), +var fs = require('fs'), path = require('path'), - fs = require('fs'), + async = require('async'), winston = require('winston'), - nconf = require('nconf'); + nconf = require('nconf'), + + utils = require('./../public/src/utils'), + db = require('./database'), + plugins = require('./plugins'); + (function (Meta) { Meta.config = {}; @@ -15,33 +17,34 @@ var utils = require('./../public/src/utils.js'), delete Meta.config; Meta.configs.list(function (err, config) { - if (!err) { - Meta.config = config; - callback(); - } else { + if(err) { winston.error(err); + return callback(err); } + + Meta.config = config; + callback(); }); }, list: function (callback) { - RDB.hgetall('config', function (err, config) { - if (!err) { - config = config || {}; - config.status = 'ok'; - callback(err, config); - } else { - callback(new Error('could-not-read-config')); + db.getObject('config', function (err, config) { + if(err) { + return callback(new Error('could-not-read-config')); } + + config = config || {}; + config.status = 'ok'; + callback(err, config); }); }, get: function (field, callback) { - RDB.hget('config', field, callback); + db.getObjectField('config', field, callback); }, getFields: function (fields, callback) { - RDB.hmgetObject('config', fields, callback); + db.getObjectFields('config', fields, callback); }, set: function (field, value, callback) { - RDB.hset('config', field, value, function (err, res) { + db.setObjectField(field, value, function(err, res) { if (callback) { if(!err && Meta.config) Meta.config[field] = value; @@ -59,7 +62,7 @@ var utils = require('./../public/src/utils.js'), }); }, remove: function (field) { - RDB.hdel('config', field); + db.deleteObjectField('config', field); } }; @@ -125,7 +128,7 @@ var utils = require('./../public/src/utils.js'), themeData['theme:staticDir'] = config.staticDir ? config.staticDir : ''; themeData['theme:templates'] = config.templates ? config.templates : ''; - RDB.hmset('config', themeData, next); + db.setObject('config', themeData, next); } ], function(err) { callback(err); @@ -134,7 +137,7 @@ var utils = require('./../public/src/utils.js'), case 'bootswatch': themeData['theme:src'] = data.src; - RDB.hmset('config', themeData, callback); + db.setObject('config', themeData, callback); break; } } @@ -256,11 +259,16 @@ var utils = require('./../public/src/utils.js'), }), minified; - if (process.env.NODE_ENV === 'development') winston.info('Minifying client-side libraries'); + if (process.env.NODE_ENV === 'development') { + winston.info('Minifying client-side libraries'); + } + minified = uglifyjs.minify(jsPaths); fs.writeFile(Meta.js.minFile, minified.code, function (err) { if (!err) { - if (process.env.NODE_ENV === 'development') winston.info('Minified client-side libraries'); + if (process.env.NODE_ENV === 'development') { + winston.info('Minified client-side libraries'); + } callback(); } else { winston.error('Problem minifying client-side libraries, exiting.'); @@ -272,23 +280,7 @@ var utils = require('./../public/src/utils.js'), Meta.db = { getFile: function (callback) { - var multi = RDB.multi(); - - multi.config('get', 'dir'); - multi.config('get', 'dbfilename'); - multi.exec(function (err, results) { - if (err) { - return callback(err); - } else { - results = results.reduce(function (memo, config) { - memo[config[0]] = config[1]; - return memo; - }, {}); - - var dbFile = path.join(results.dir, results.dbfilename); - callback(null, dbFile); - } - }); + db.getFileName(callback); } }; }(exports)); \ No newline at end of file diff --git a/src/plugins.js b/src/plugins.js index 9de194034f..9a7142366e 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -1,9 +1,10 @@ var fs = require('fs'), path = require('path'), - RDB = require('./redis.js'), async = require('async'), winston = require('winston'), eventEmitter = require('events').EventEmitter, + db = require('./database'), + plugins = { libraries: {}, loadedHooks: {}, From 347d6c27687c32e991e4bb305abab428c170896d Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Mon, 2 Dec 2013 16:23:14 -0500 Subject: [PATCH 04/83] moved filename function into redis --- src/database/redis.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/database/redis.js b/src/database/redis.js index bd6597f3c5..a52ad7ec1c 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -48,6 +48,27 @@ // // Exported functions // + module.getFileName = function(callback) { + var multi = redisClient.multi(); + + multi.config('get', 'dir'); + multi.config('get', 'dbfilename'); + multi.exec(function (err, results) { + if (err) { + return callback(err); + } + + results = results.reduce(function (memo, config) { + memo[config[0]] = config[1]; + return memo; + }, {}); + + var dbFile = path.join(results.dir, results.dbfilename); + callback(null, dbFile); + }); + } + + module.setObject = function(key, data, callback) { redisClient.hmset(key, data, callback); } From 636551d2e939f2fb98ca159313a942b8e6692f49 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Mon, 2 Dec 2013 16:35:32 -0500 Subject: [PATCH 05/83] plugins.js uses db, added some set methods to redis.js --- src/database/redis.js | 15 +++++++++++++++ src/plugins.js | 6 +++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/database/redis.js b/src/database/redis.js index a52ad7ec1c..827491966f 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -116,6 +116,21 @@ } + module.setAdd = function(key, value, callback) { + redisClient.sadd(key, value, callback); + } + + module.setRemove = function(key, value, callback) { + redisClient.srem(key, value, callback); + } + + module.isSetMember = function(key, value, callback) { + redisClient.sismember(key, value, callback); + } + + module.getSetMembers = function(key, callback) { + redisClient.smembers(key, callback); + } diff --git a/src/plugins.js b/src/plugins.js index 9a7142366e..7dbc0a6bdd 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -46,7 +46,7 @@ var fs = require('fs'), // Read the list of activated plugins and require their libraries async.waterfall([ function(next) { - RDB.smembers('plugins:active', next); + db.getSetMembers('plugins:active', next); }, function(plugins, next) { if (plugins && Array.isArray(plugins) && plugins.length > 0) { @@ -230,7 +230,7 @@ var fs = require('fs'), } }, isActive: function(id, callback) { - RDB.sismember('plugins:active', id, callback); + db.isSetMember('plugins:active', id, callback); }, toggleActive: function(id, callback) { this.isActive(id, function(err, active) { @@ -239,7 +239,7 @@ var fs = require('fs'), return; } - RDB[(active ? 'srem' : 'sadd')]('plugins:active', id, function(err, success) { + db[(active ? 'setRemove' : 'setAdd')]('plugins:active', id, function(err, success) { if (err) { if (global.env === 'development') winston.info('[plugins] Could not toggle active state on plugin \'' + id + '\''); return; From c9308efbec6877963101177ce9548942737affed Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Mon, 2 Dec 2013 17:10:26 -0500 Subject: [PATCH 06/83] more work --- app.js | 33 +++++++++++++++++---------------- src/database/redis.js | 17 +++++++++++++++-- src/user.js | 39 ++++++++++++++++++++++----------------- src/webserver.js | 35 +++++++++++++++++++++++------------ 4 files changed, 77 insertions(+), 47 deletions(-) diff --git a/app.js b/app.js index 7507ee4ac1..b1e27a6c9b 100644 --- a/app.js +++ b/app.js @@ -74,25 +74,26 @@ } meta.configs.init(function () { - // Initial setup for Redis & Reds - var reds = require('reds'), - RDB = require('./src/redis.js'); - reds.createClient = function () { - return reds.client || (reds.client = RDB); - }; - - var templates = require('./public/src/templates.js'), - translator = require('./public/src/translator.js'), - webserver = require('./src/webserver.js'), + // + // TODO : figure out reds search after dbal is complete + // + //var reds = require('reds'), + // db = require('./src/database'); + /*reds.createClient = function () { + return reds.client || (reds.client = db); + };*/ + + var templates = require('./public/src/templates'), + translator = require('./public/src/translator'), + webserver = require('./src/webserver'), SocketIO = require('socket.io').listen(global.server, { log: false, transports: ['websocket', 'xhr-polling', 'jsonp-polling', 'flashsocket'], 'browser client minification': true}), - websockets = require('./src/websockets.js'), - posts = require('./src/posts.js'), + websockets = require('./src/websockets'), plugins = require('./src/plugins'), // Don't remove this - plugins initializes itself - Notifications = require('./src/notifications'), - Upgrade = require('./src/upgrade'); + notifications = require('./src/notifications'), + upgrade = require('./src/upgrade'); - Upgrade.check(function(schema_ok) { + upgrade.check(function(schema_ok) { if (schema_ok || nconf.get('check-schema') === false) { websockets.init(SocketIO); @@ -117,7 +118,7 @@ templates.ready(webserver.init); }); - Notifications.init(); + notifications.init(); } else { winston.warn('Your NodeBB schema is out-of-date. Please run the following command to bring your dataset up to spec:'); winston.warn(' node app --upgrade'); diff --git a/src/database/redis.js b/src/database/redis.js index 827491966f..0f7fa840c4 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -111,10 +111,13 @@ redisClient.hdel(key, field, callback); } - module.incrObjectField = function(key, field, value, callback) { - redisClient.hincrby(key, field, value, callback); + module.incrObjectField = function(key, field, callback) { + redisClient.hincrby(key, field, 1, callback); } + module.incrObjectFieldBy = function(key, field, value, callback) { + redisClient.hincrby(key, field, value, callback); + } module.setAdd = function(key, value, callback) { redisClient.sadd(key, value, callback); @@ -132,6 +135,16 @@ redisClient.smembers(key, callback); } + module.sortedSetAdd = function(key, score, value, callback) { + redisClient.zadd(key, score, value, callback); + } + + module.sortedSetRemove = function(key, value, callback) { + redisClient.zrem(key, value, callback); + } + + + }(exports)); diff --git a/src/user.js b/src/user.js index bfb312db91..92ba8ac42e 100644 --- a/src/user.js +++ b/src/user.js @@ -10,7 +10,7 @@ var bcrypt = require('bcrypt'), utils = require('./../public/src/utils'), plugins = require('./plugins'), - RDB = require('./redis'), + db = require('./database'), meta = require('./meta'), emailjsServer = emailjs.server.connect(meta.config['email:smtp:host'] || '127.0.0.1'), Groups = require('./groups'), @@ -65,16 +65,18 @@ var bcrypt = require('bcrypt'), } ], function(err, results) { if (err) { - return callback(err, null); + return callback(err); } - RDB.incr('global:next_user_id', function(err, uid) { - RDB.handle(err); + db.incrObjectField('global', 'nextUserId', function(err, uid) { + if(err) { + return callback(err); + } var gravatar = User.createGravatarURLFromEmail(email); var timestamp = Date.now(); - RDB.hmset('user:' + uid, { + db.setObject('user:' + uid, { 'uid': uid, 'username': username, 'userslug': userslug, @@ -96,20 +98,20 @@ var bcrypt = require('bcrypt'), 'showemail': 0 }); - RDB.hset('username:uid', username, uid); - RDB.hset('userslug:uid', userslug, uid); + db.setObjectField('username:uid', username, uid); + db.setObjectField('userslug:uid', userslug, uid); if (email !== undefined) { - RDB.hset('email:uid', email, uid); + db.setObjectField('email:uid', email, uid); User.sendConfirmationEmail(email); } plugins.fireHook('action:user.create', {uid: uid, username: username, email: email, picture: gravatar, timestamp: timestamp}); - RDB.incr('usercount'); + db.incrObjectField('global', 'usercount'); - RDB.zadd('users:joindate', timestamp, uid); - RDB.zadd('users:postcount', 0, uid); - RDB.zadd('users:reputation', 0, uid); + db.sortedSetAdd('users:joindate', timestamp, uid); + db.sortedSetAdd('users:postcount', 0, uid); + db.sortedSetAdd('users:reputation', 0, uid); userSearch.index(username, uid); @@ -134,11 +136,11 @@ var bcrypt = require('bcrypt'), }; User.getUserField = function(uid, field, callback) { - RDB.hget('user:' + uid, field, callback); + db.getObjectField('user:' + uid, field, callback); }; User.getUserFields = function(uid, fields, callback) { - RDB.hmgetObject('user:' + uid, fields, callback); + db.getObjectFields('user:' + uid, fields, callback); }; User.getMultipleUserFields = function(uids, fields, callback) { @@ -168,7 +170,10 @@ var bcrypt = require('bcrypt'), }; User.getUserData = function(uid, callback) { - RDB.hgetall('user:' + uid, function(err, data) { + db.getObject('user:' + uid, function(err, data) { + if(err) { + return callback(err); + } if (data && data.password) { delete data.password; @@ -253,8 +258,8 @@ var bcrypt = require('bcrypt'), return next(err); } - RDB.hdel('email:uid', userData.email); - RDB.hset('email:uid', data.email, uid); + db.deleteObjectField('email:uid', userData.email); + db.setObjectField('email:uid', data.email, uid); User.setUserField(uid, field, data[field]); if (userData.picture !== userData.uploadedpicture) { returnData.picture = gravatarpicture; diff --git a/src/webserver.js b/src/webserver.js index 716bd5dcfe..60c5e87e62 100644 --- a/src/webserver.js +++ b/src/webserver.js @@ -14,7 +14,7 @@ var path = require('path'), pkg = require('../package.json'), utils = require('../public/src/utils'), - RDB = require('./redis'), + db = require('./database'), user = require('./user'), categories = require('./categories'), posts = require('./posts'), @@ -140,7 +140,9 @@ var path = require('path'), })); app.use(express.bodyParser()); // Puts POST vars in request.body app.use(express.cookieParser()); // If you want to parse cookies (res.cookies) - app.use(express.session({ + + // TODO : this uses redis + /*app.use(express.session({ store: new RedisStore({ client: RDB, ttl: 60 * 60 * 24 * 30 @@ -150,7 +152,16 @@ var path = require('path'), cookie: { maxAge: 60 * 60 * 24 * 30 * 1000 // 30 days } + }));*/ + + app.use(express.cookieSession({ + secret: nconf.get('secret'), + key: 'express.sid', + cookie: { + maxAge: 60 * 60 * 24 * 30 * 1000 // 30 days + } })); + app.use(express.csrf()); // Local vars, other assorted setup @@ -172,33 +183,33 @@ var path = require('path'), function(next) { async.parallel([ function(next) { - // Theme configuration - RDB.hmget('config', 'theme:type', 'theme:id', 'theme:staticDir', 'theme:templates', function(err, themeData) { - var themeId = (themeData[1] || 'nodebb-theme-vanilla'); + + db.getObjectFields('config', ['theme:type', 'theme:id', 'theme:staticDir', 'theme:templates'], function(err, themeData) { + var themeId = (themeData['theme:id'] || 'nodebb-theme-vanilla'); // Detect if a theme has been selected, and handle appropriately - if (!themeData[0] || themeData[0] === 'local') { + if (!themeData['theme:type'] || themeData['theme:type'] === 'local') { // Local theme if (process.env.NODE_ENV === 'development') { winston.info('[themes] Using theme ' + themeId); } // Theme's static directory - if (themeData[2]) { - app.use('/css/assets', express.static(path.join(__dirname, '../node_modules', themeData[1], themeData[2]), { + if (themeData['theme:staticDir']) { + app.use('/css/assets', express.static(path.join(__dirname, '../node_modules', themeData['theme:id'], themeData['theme:staticDir']), { maxAge: app.enabled('cache') ? 5184000000 : 0 })); if (process.env.NODE_ENV === 'development') { - winston.info('Static directory routed for theme: ' + themeData[1]); + winston.info('Static directory routed for theme: ' + themeData['theme:id']); } } - if (themeData[3]) { - app.use('/templates', express.static(path.join(__dirname, '../node_modules', themeData[1], themeData[3]), { + if (themeData['theme:templates']) { + app.use('/templates', express.static(path.join(__dirname, '../node_modules', themeData['theme:id'], themeData['theme:templates']), { maxAge: app.enabled('cache') ? 5184000000 : 0 })); if (process.env.NODE_ENV === 'development') { - winston.info('Custom templates directory routed for theme: ' + themeData[1]); + winston.info('Custom templates directory routed for theme: ' + themeData['theme:id']); } } From e32d2309744021b3f8ea6ed7292de79f07f5e11c Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 17:48:21 -0500 Subject: [PATCH 07/83] format --- src/notifications.js | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/notifications.js b/src/notifications.js index 801995c4d6..06786119dd 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -16,7 +16,7 @@ var RDB = require('./redis'), } new cron('0 0 * * *', Notifications.prune, null, true); }; - + Notifications.get = function(nid, uid, callback) { RDB.multi() .hmget('notifications:' + nid, 'text', 'score', 'path', 'datetime', 'uniqueId') @@ -41,7 +41,7 @@ var RDB = require('./redis'), }); }); }; - + Notifications.create = function(text, path, uniqueId, callback) { /** * uniqueId is used solely to override stale nids. @@ -63,7 +63,7 @@ var RDB = require('./redis'), }); }); }; - + function destroy(nid) { var multi = RDB.multi(); @@ -77,9 +77,11 @@ var RDB = require('./redis'), } }); } - + Notifications.push = function(nid, uids, callback) { - if (!Array.isArray(uids)) uids = [uids]; + if (!Array.isArray(uids)) { + uids = [uids]; + } var numUids = uids.length, x; @@ -102,7 +104,7 @@ var RDB = require('./redis'), } }); }; - + function remove_by_uniqueId(uniqueId, uid, callback) { async.parallel([ function(next) { @@ -149,7 +151,7 @@ var RDB = require('./redis'), } }); } - + Notifications.mark_read = function(nid, uid, callback) { if (parseInt(uid) > 0) { Notifications.get(nid, uid, function(notif_data) { @@ -161,7 +163,7 @@ var RDB = require('./redis'), }); } } - + Notifications.mark_read_multiple = function(nids, uid, callback) { if (!Array.isArray(nids) && parseInt(nids, 10) > 0) { nids = [nids]; @@ -179,7 +181,7 @@ var RDB = require('./redis'), } }); }; - + Notifications.mark_all_read = function(uid, callback) { RDB.zrange('uid:' + uid + ':notifications:unread', 0, 10, function(err, nids) { if (err) { @@ -195,7 +197,7 @@ var RDB = require('./redis'), } }); }; - + Notifications.prune = function(cutoff) { if (process.env.NODE_ENV === 'development') { winston.info('[notifications.prune] Removing expired notifications from the database.'); @@ -269,6 +271,6 @@ var RDB = require('./redis'), } }); }; - + }(exports)); From 3775c8e50a3943d55272f6f4d6e4350127c5b483 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 19:40:11 -0500 Subject: [PATCH 08/83] tons more changes --- src/admin/categories.js | 6 +- src/database/redis.js | 57 +++++++++- src/groups.js | 89 ++++++++-------- src/login.js | 88 +++++++++------ src/user.js | 230 ++++++++++++++++++++++------------------ 5 files changed, 287 insertions(+), 183 deletions(-) diff --git a/src/admin/categories.js b/src/admin/categories.js index 494a767e9c..333a22cf70 100644 --- a/src/admin/categories.js +++ b/src/admin/categories.js @@ -1,4 +1,4 @@ -var RDB = require('./../redis'), +var db = require('./../database'), utils = require('./../../public/src/utils'), categories = require('./../categories'); @@ -11,12 +11,12 @@ var RDB = require('./../redis'), var category = modified[cid]; for (var key in category) { - RDB.hset('category:' + cid, key, category[key]); + db.setObjectField('category:' + cid, key, category[key]); if (key == 'name') { // reset slugs if name is updated var slug = cid + '/' + utils.slugify(category[key]); - RDB.hset('category:' + cid, 'slug', slug); + db.setObjectField('category:' + cid, 'slug', slug); } } diff --git a/src/database/redis.js b/src/database/redis.js index 0f7fa840c4..9492470e0c 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -68,13 +68,26 @@ }); } + // key + + module.exists = function(key, callback) { + redisClient.exists(key, function(err, exists) { + callback(err, exists === 1); + }); + } + + module.delete = function(key, callback) { + redisClient.del(key, callback); + } + + //hashes module.setObject = function(key, data, callback) { redisClient.hmset(key, data, callback); } - module.setObjectField = function(key, field, callback) { - redisClient.hset(key, field, callback) + module.setObjectField = function(key, field, value, callback) { + redisClient.hset(key, field, value, callback) } module.getObject = function(key, callback) { @@ -107,6 +120,16 @@ }); } + module.getObjectValues = function(key, callback) { + redisClient.hvals(key, callback); + } + + module.isObjectField = function(key, field, callback) { + redisClient.hexists(key, field, function(err, exists) { + callback(err, exists === 1); + }); + } + module.deleteObjectField = function(key, field, callback) { redisClient.hdel(key, field, callback); } @@ -119,6 +142,9 @@ redisClient.hincrby(key, field, value, callback); } + + // sets + module.setAdd = function(key, value, callback) { redisClient.sadd(key, value, callback); } @@ -135,6 +161,12 @@ redisClient.smembers(key, callback); } + module.setCount = function(key, callback) { + redisClient.scard(key, callback); + } + + // sorted sets + module.sortedSetAdd = function(key, score, value, callback) { redisClient.zadd(key, score, value, callback); } @@ -143,9 +175,30 @@ redisClient.zrem(key, value, callback); } + module.getSortedSetRange = function(key, start, stop, callback) { + redisClient.zrange(set, start, stop, callback); + } + + module.getSortedSetRevRange = function(key, start, stop, callback) { + redisClient.zrevrange(set, start, stop, callback); + } + + module.sortedSetCount = function(key, min, max, callback) { + redisClient.zcount(key, min, max, callback); + } + // lists + module.listPrepend = function(key, value, callback) { + redisClient.lpush(key, value, callback); + } + module.listAppend = function(key, value, callback) { + redisClient.rpush(key, value, callback); + } + module.getListRange = function(key, start, stop, callback) { + redisClient.lrange(key, start, stop, callback); + } }(exports)); diff --git a/src/groups.js b/src/groups.js index 058b8fdd08..98faf0bcbb 100644 --- a/src/groups.js +++ b/src/groups.js @@ -2,11 +2,11 @@ "use strict"; var async = require('async'), - User = require('./user'), - RDB = RDB || require('./redis'); + user = require('./user'), + db = require('./database'); Groups.list = function(options, callback) { - RDB.hvals('group:gid', function (err, gids) { + db.getObjectValues('group:gid', function (err, gids) { if (gids.length > 0) { async.map(gids, function (gid, next) { Groups.get(gid, { @@ -31,17 +31,17 @@ Groups.get = function(gid, options, callback) { async.parallel({ base: function (next) { - RDB.hgetall('gid:' + gid, next); + db.getObject('gid:' + gid, next); }, users: function (next) { - RDB.smembers('gid:' + gid + ':members', function (err, uids) { + db.getSetMembers('gid:' + gid + ':members', function (err, uids) { if (options.expand) { if (err) { return next(err); } async.map(uids, function (uid, next) { - User.getUserData(uid, next); + user.getUserData(uid, next); }, function (err, users) { next(err, users); }); @@ -75,19 +75,19 @@ }; Groups.isDeleted = function(gid, callback) { - RDB.hget('gid:' + gid, 'deleted', function(err, deleted) { + db.getObjectField('gid:' + gid, 'deleted', function(err, deleted) { callback(err, deleted === '1'); }); }; Groups.getGidFromName = function(name, callback) { - RDB.hget('group:gid', name, callback); + db.getObjectField('group:gid', name, callback); }; Groups.isMember = function(uid, gid, callback) { Groups.isDeleted(gid, function(err, deleted) { if (!deleted) { - RDB.sismember('gid:' + gid + ':members', uid, callback); + db.isSetMember('gid:' + gid + ':members', uid, callback); } else { callback(err, false); } @@ -107,7 +107,7 @@ }; Groups.isEmpty = function(gid, callback) { - RDB.scard('gid:' + gid + ':members', function(err, numMembers) { + db.setCount('gid:' + gid + ':members', function(err, numMembers) { callback(err, numMembers === 0); }); }; @@ -125,7 +125,7 @@ Groups.exists = function(name, callback) { async.parallel({ exists: function(next) { - RDB.hexists('group:gid', name, next); + db.isObjectField('group:gid', name, next); }, deleted: function(next) { Groups.getGidFromName(name, function(err, gid) { @@ -144,19 +144,14 @@ Groups.exists(name, function (err, exists) { if (!exists) { - RDB.incr('next_gid', function (err, gid) { - RDB.multi() - .hset('group:gid', name, gid) - .hmset('gid:' + gid, { - gid: gid, - name: name, - description: description, - deleted: '0', - hidden: '0' - }) - .exec(function (err) { + db.incrObjectField('global', 'nextGid', function (err, gid) { + db.setObjectField('group:gid', name, gid, function(err) { + db.setObject('gid:' + gid, {}, function(err) { + Groups.get(gid, {}, callback); + }); + }); }); } else { callback(new Error('group-exists')); @@ -171,22 +166,24 @@ }; Groups.update = function(gid, values, callback) { - RDB.exists('gid:' + gid, function (err, exists) { + db.exists('gid:' + gid, function (err, exists) { console.log('exists?', gid, exists, values); if (!err && exists) { - RDB.hmset('gid:' + gid, values, callback); + db.setObject('gid:' + gid, values, callback); } else { - if (callback) callback(new Error('gid-not-found')); + if (callback) { + callback(new Error('gid-not-found')); + } } }); }; Groups.destroy = function(gid, callback) { - RDB.hset('gid:' + gid, 'deleted', '1', callback); + db.setObjectField('gid:' + gid, 'deleted', '1', callback); }; Groups.join = function(gid, uid, callback) { - RDB.sadd('gid:' + gid + ':members', uid, callback); + db.setAdd('gid:' + gid + ':members', uid, callback); }; Groups.joinByGroupName = function(groupName, uid, callback) { @@ -210,7 +207,7 @@ }; Groups.leave = function(gid, uid, callback) { - RDB.srem('gid:' + gid + ':members', uid, callback); + db.setRemove('gid:' + gid + ':members', uid, callback); }; Groups.leaveByGroupName = function(groupName, uid, callback) { @@ -225,30 +222,36 @@ Groups.prune = function(callback) { // Actually deletes groups (with the deleted flag) from the redis database - RDB.hvals('group:gid', function (err, gids) { - var multi = RDB.multi(), - groupsDeleted = 0; + db.getObjectValues('group:gid', function (err, gids) { + var groupsDeleted = 0; async.each(gids, function(gid, next) { Groups.get(gid, {}, function(err, groupObj) { - if (!err && groupObj.deleted === '1') { - multi.hdel('group:gid', groupObj.name); - multi.del('gid:' + gid); - groupsDeleted++; + if(err) { + return next(err); } - next(null); + if (groupObj.deleted === '1') { + + db.deleteObjectField('group:gid', groupObj.name, function(err) { + db.delete('gid:' + gid, function(err) { + groupsDeleted++; + next(null); + }); + }); + } else { + next(null); + } }); }, function(err) { - multi.exec(function(err) { - if (!err && process.env.NODE_ENV === 'development') { - winston.info('[groups.prune] Pruned ' + groupsDeleted + ' deleted groups from Redis'); - } - callback(err); - }); + if (!err && process.env.NODE_ENV === 'development') { + winston.info('[groups.prune] Pruned ' + groupsDeleted + ' deleted groups from Redis'); + } + + callback(err); }); }); }; - + }(module.exports)); diff --git a/src/login.js b/src/login.js index 51f3ed5472..d63355ad32 100644 --- a/src/login.js +++ b/src/login.js @@ -58,7 +58,11 @@ var user = require('./user.js'), } Login.loginViaTwitter = function(twid, handle, photos, callback) { - user.getUidByTwitterId(twid, function(uid) { + user.getUidByTwitterId(twid, function(err, uid) { + if(err) { + return callback(err); + } + if (uid !== null) { // Existing User callback(null, { @@ -67,32 +71,36 @@ var user = require('./user.js'), } else { // New User user.create(handle, undefined, undefined, function(err, uid) { - if (err !== null) { - callback(err); - } else { - // Save twitter-specific information to the user - user.setUserField(uid, 'twid', twid); - RDB.hset('twid:uid', twid, uid); - - // Save their photo, if present - if (photos && photos.length > 0) { - var photoUrl = photos[0].value; - photoUrl = path.dirname(photoUrl) + '/' + path.basename(photoUrl, path.extname(photoUrl)).slice(0, -6) + 'bigger' + path.extname(photoUrl); - user.setUserField(uid, 'uploadedpicture', photoUrl); - user.setUserField(uid, 'picture', photoUrl); - } + if(err) { + return callback(err); + } - callback(null, { - uid: uid - }); + // Save twitter-specific information to the user + user.setUserField(uid, 'twid', twid); + RDB.hset('twid:uid', twid, uid); + + // Save their photo, if present + if (photos && photos.length > 0) { + var photoUrl = photos[0].value; + photoUrl = path.dirname(photoUrl) + '/' + path.basename(photoUrl, path.extname(photoUrl)).slice(0, -6) + 'bigger' + path.extname(photoUrl); + user.setUserField(uid, 'uploadedpicture', photoUrl); + user.setUserField(uid, 'picture', photoUrl); } + + callback(null, { + uid: uid + }); }); } }); } Login.loginViaGoogle = function(gplusid, handle, email, callback) { - user.getUidByGoogleId(gplusid, function(uid) { + user.getUidByGoogleId(gplusid, function(err, uid) { + if(err) { + return callback(err); + } + if (uid !== null) { // Existing User callback(null, { @@ -109,21 +117,33 @@ var user = require('./user.js'), }); } - user.getUidByEmail(email, function(uid) { + user.getUidByEmail(email, function(err, uid) { + if(err) { + return callback(err); + } + if (!uid) { user.create(handle, undefined, email, function(err, uid) { - if (err !== null) { - callback(err); - } else success(uid); + if(err) { + return callback(err); + } + + success(uid); }); - } else success(uid); // Existing account -- merge + } else { + success(uid); // Existing account -- merge + } }); } }); } Login.loginViaFacebook = function(fbid, name, email, callback) { - user.getUidByFbid(fbid, function(uid) { + user.getUidByFbid(fbid, function(err, uid) { + if(err) { + return callback(err); + } + if (uid !== null) { // Existing User callback(null, { @@ -140,14 +160,22 @@ var user = require('./user.js'), }); } - user.getUidByEmail(email, function(uid) { + user.getUidByEmail(email, function(err, uid) { + if(err) { + return callback(err); + } + if (!uid) { user.create(name, undefined, email, function(err, uid) { - if (err !== null) { - callback(err); - } else success(uid); + if(err) { + return callback(err); + } + + success(uid); }); - } else success(uid); // Existing account -- merge + } else { + success(uid); // Existing account -- merge + } }); } }); diff --git a/src/user.js b/src/user.js index 92ba8ac42e..d932d235f5 100644 --- a/src/user.js +++ b/src/user.js @@ -13,7 +13,7 @@ var bcrypt = require('bcrypt'), db = require('./database'), meta = require('./meta'), emailjsServer = emailjs.server.connect(meta.config['email:smtp:host'] || '127.0.0.1'), - Groups = require('./groups'), + groups = require('./groups'), notifications = require('./notifications'), topics = require('./topics'); @@ -287,7 +287,7 @@ var bcrypt = require('bcrypt'), }; User.isEmailAvailable = function(email, callback) { - RDB.hexists('email:uid', email, function(err, exists) { + db.isObjectField('email:uid', email, function(err, exists) { callback(err, !exists); }); }; @@ -321,25 +321,25 @@ var bcrypt = require('bcrypt'), }; User.setUserField = function(uid, field, value, callback) { - RDB.hset('user:' + uid, field, value, callback); + db.setObjectField('user:' + uid, field, value, callback); }; User.setUserFields = function(uid, data, callback) { - RDB.hmset('user:' + uid, data, callback); + db.setObject('user:' + uid, data, callback); }; User.incrementUserFieldBy = function(uid, field, value, callback) { - RDB.hincrby('user:' + uid, field, value, callback); + db.incrObjectFieldBy('user:' + uid, field, value, callback); }; User.decrementUserFieldBy = function(uid, field, value, callback) { - RDB.hincrby('user:' + uid, field, -value, callback); + db.incrObjectFieldBy('user:' + uid, field, -value, callback); }; User.getUsers = function(set, start, stop, callback) { var data = []; - RDB.zrevrange(set, start, stop, function(err, uids) { + db.getSortedSetRevRange(set, start, stop, function(err, uids) { if (err) { return callback(err, null); } @@ -360,7 +360,6 @@ var bcrypt = require('bcrypt'), callback(err, data); }); }); - }; User.createGravatarURLFromEmail = function(email) { @@ -433,7 +432,7 @@ var bcrypt = require('bcrypt'), User.addPostIdToUser(uid, pid); User.incrementUserFieldBy(uid, 'postcount', 1, function(err, newpostcount) { - RDB.zadd('users:postcount', newpostcount, uid); + db.sortedSetAdd('users:postcount', newpostcount, uid); }); User.setUserField(uid, 'lastposttime', timestamp); @@ -442,15 +441,15 @@ var bcrypt = require('bcrypt'), }; User.addPostIdToUser = function(uid, pid) { - RDB.lpush('uid:' + uid + ':posts', pid); + db.listPrepend('uid:' + uid + ':posts', pid); }; User.addTopicIdToUser = function(uid, tid) { - RDB.lpush('uid:' + uid + ':topics', tid); + db.listPrepend('uid:' + uid + ':topics', tid); }; - User.getPostIds = function(uid, start, end, callback) { - RDB.lrange('uid:' + uid + ':posts', start, end, function(err, pids) { + User.getPostIds = function(uid, start, stop, callback) { + db.getListRange('uid:' + uid + ':posts', start, stop, function(err, pids) { if (!err) { if (pids && pids.length) { callback(pids); @@ -464,51 +463,12 @@ var bcrypt = require('bcrypt'), }); }; - User.sendConfirmationEmail = function(email) { - if (meta.config['email:smtp:host'] && meta.config['email:smtp:port'] && meta.config['email:from']) { - var confirm_code = utils.generateUUID(), - confirm_link = nconf.get('url') + 'confirm/' + confirm_code, - confirm_email = global.templates['emails/header'] + global.templates['emails/email_confirm'].parse({ - 'CONFIRM_LINK': confirm_link - }) + global.templates['emails/footer'], - confirm_email_plaintext = global.templates['emails/email_confirm_plaintext'].parse({ - 'CONFIRM_LINK': confirm_link - }); - - // Email confirmation code - var expiry_time = 60 * 60 * 2, // Expire after 2 hours - email_key = 'email:' + email + ':confirm', - confirm_key = 'confirm:' + confirm_code + ':email'; - - RDB.set(email_key, confirm_code); - RDB.expire(email_key, expiry_time); - RDB.set(confirm_key, email); - RDB.expire(confirm_key, expiry_time); - - // Send intro email w/ confirm code - var message = emailjs.message.create({ - text: confirm_email_plaintext, - from: meta.config['email:from'] || 'localhost@example.org', - to: email, - subject: '[NodeBB] Registration Email Verification', - attachment: [{ - data: confirm_email, - alternative: true - }] - }); - emailjsServer.send(message, function(err, success) { - if (err) { - console.log(err); - } - }); - } - }; User.follow = function(uid, followid, callback) { - RDB.sadd('following:' + uid, followid, function(err, data) { + db.setAdd('following:' + uid, followid, function(err, data) { if (!err) { - RDB.sadd('followers:' + followid, uid, function(err, data) { + db.setAdd('followers:' + followid, uid, function(err, data) { if (!err) { callback(true); } else { @@ -524,7 +484,7 @@ var bcrypt = require('bcrypt'), }; User.unfollow = function(uid, unfollowid, callback) { - RDB.srem('following:' + uid, unfollowid, function(err, data) { + db.setRemove('following:' + uid, unfollowid, function(err, data) { if (!err) { RDB.srem('followers:' + unfollowid, uid, function(err, data) { callback(data); @@ -536,7 +496,7 @@ var bcrypt = require('bcrypt'), }; User.getFollowing = function(uid, callback) { - RDB.smembers('following:' + uid, function(err, userIds) { + db.getSetMembers('following:' + uid, function(err, userIds) { if (!err) { User.getDataForUsers(userIds, callback); } else { @@ -546,7 +506,7 @@ var bcrypt = require('bcrypt'), }; User.getFollowers = function(uid, callback) { - RDB.smembers('followers:' + uid, function(err, userIds) { + db.getSetMembers('followers:' + uid, function(err, userIds) { if (!err) { User.getDataForUsers(userIds, callback); } else { @@ -556,7 +516,7 @@ var bcrypt = require('bcrypt'), }; User.getFollowingCount = function(uid, callback) { - RDB.smembers('following:' + uid, function(err, userIds) { + db.getSetMembers('following:' + uid, function(err, userIds) { if (err) { console.log(err); } else { @@ -569,7 +529,7 @@ var bcrypt = require('bcrypt'), }; User.getFollowerCount = function(uid, callback) { - RDB.smembers('followers:' + uid, function(err, userIds) { + db.getSetMembers('followers:' + uid, function(err, userIds) { if(err) { console.log(err); } else { @@ -608,7 +568,7 @@ var bcrypt = require('bcrypt'), User.sendPostNotificationToFollowers = function(uid, tid, pid) { User.getUserField(uid, 'username', function(err, username) { - RDB.smembers('followers:' + uid, function(err, followers) { + db.getSetMembers('followers:' + uid, function(err, followers) { topics.getTopicField(tid, 'slug', function(err, slug) { var message = '' + username + ' made a new post'; @@ -621,7 +581,7 @@ var bcrypt = require('bcrypt'), }; User.isFollowing = function(uid, theirid, callback) { - RDB.sismember('following:' + uid, theirid, function(err, data) { + db.isSetMember('following:' + uid, theirid, function(err, data) { if (!err) { callback(data === 1); } else { @@ -637,8 +597,10 @@ var bcrypt = require('bcrypt'), }; User.count = function(socket) { - RDB.get('usercount', function(err, count) { - RDB.handle(err); + db.getObjectField('global', 'usercount', function(err, count) { + if(err) { + return; + } socket.emit('user.count', { count: count ? count : 0 @@ -647,11 +609,11 @@ var bcrypt = require('bcrypt'), }; User.getUidByUsername = function(username, callback) { - RDB.hget('username:uid', username, callback); + db.getObjectField('username:uid', username, callback); }; User.getUidByUserslug = function(userslug, callback) { - RDB.hget('userslug:uid', userslug, callback); + db.getObjectField('userslug:uid', userslug, callback); }; User.getUsernamesByUids = function(uids, callback) { @@ -693,51 +655,53 @@ var bcrypt = require('bcrypt'), }; User.getUidByEmail = function(email, callback) { - RDB.hget('email:uid', email, function(err, data) { + db.getObjectField('email:uid', email, function(err, data) { if (err) { - RDB.handle(err); + return callback(err); } - callback(data); + callback(null, data); }); }; User.getUidByTwitterId = function(twid, callback) { - RDB.hget('twid:uid', twid, function(err, uid) { + db.getObjectField('twid:uid', twid, function(err, uid) { if (err) { - RDB.handle(err); + return callback(err); } - callback(uid); + callback(null, uid); }); }; User.getUidByGoogleId = function(gplusid, callback) { - RDB.hget('gplusid:uid', gplusid, function(err, uid) { + db.getObjectField('gplusid:uid', gplusid, function(err, uid) { if (err) { - RDB.handle(err); + return callback(err); } - callback(uid); + callback(null, uid); }); }; User.getUidByFbid = function(fbid, callback) { - RDB.hget('fbid:uid', fbid, function(err, uid) { + db.getObjectField('fbid:uid', fbid, function(err, uid) { if (err) { - RDB.handle(err); + return callback(err); } - callback(uid); + callback(null, uid); }); }; User.isModerator = function(uid, cid, callback) { - RDB.sismember('cid:' + cid + ':moderators', uid, function(err, exists) { - RDB.handle(err); + db.isSetMember('cid:' + cid + ':moderators', uid, function(err, exists) { + if(err) { + return calback(err); + } callback(err, !! exists); }); }; User.isAdministrator = function(uid, callback) { - Groups.getGidFromName('Administrators', function(err, gid) { - Groups.isMember(uid, gid, function(err, isAdmin) { + groups.getGidFromName('Administrators', function(err, gid) { + groups.isMember(uid, gid, function(err, isAdmin) { callback(err, !! isAdmin); }); }); @@ -750,15 +714,15 @@ var bcrypt = require('bcrypt'), callback = null; } - RDB.hget('reset:uid', code, function(err, uid) { + db.getObjectField('reset:uid', code, function(err, uid) { if (err) { - RDB.handle(err); + return callback(false); } if (uid !== null) { - RDB.hget('reset:expiry', code, function(err, expiry) { + db.getObjectField('reset:expiry', code, function(err, expiry) { if (err) { - RDB.handle(err); + return callback(false); } if (expiry >= +Date.now() / 1000 | 0) { @@ -771,8 +735,8 @@ var bcrypt = require('bcrypt'), } } else { // Expired, delete from db - RDB.hdel('reset:uid', code); - RDB.hdel('reset:expiry', code); + db.deleteObjectField('reset:uid', code); + db.deleteObjectField('reset:expiry', code); if (!callback) { socket.emit('user:reset.valid', { valid: false @@ -794,12 +758,12 @@ var bcrypt = require('bcrypt'), }); }, send: function(socket, email) { - User.getUidByEmail(email, function(uid) { + User.getUidByEmail(email, function(err, uid) { if (uid !== null) { // Generate a new reset code var reset_code = utils.generateUUID(); - RDB.hset('reset:uid', reset_code, uid); - RDB.hset('reset:expiry', reset_code, (60 * 60) + new Date() / 1000 | 0); // Active for one hour + db.setObjectField('reset:uid', reset_code, uid); + db.setobjectField('reset:expiry', reset_code, (60 * 60) + new Date() / 1000 | 0); // Active for one hour var reset_link = nconf.get('url') + 'reset/' + reset_code, reset_email = global.templates['emails/reset'].parse({ @@ -847,17 +811,17 @@ var bcrypt = require('bcrypt'), commit: function(socket, code, password) { this.validate(socket, code, function(validated) { if (validated) { - RDB.hget('reset:uid', code, function(err, uid) { + db.getObjectField('reset:uid', code, function(err, uid) { if (err) { - RDB.handle(err); + return; } User.hashPassword(password, function(err, hash) { User.setUserField(uid, 'password', hash); }); - RDB.hdel('reset:uid', code); - RDB.hdel('reset:expiry', code); + db.deleteObjectField('reset:uid', code); + db.deleteObjectField('reset:expiry', code); socket.emit('user:reset.commit', { status: 'ok' @@ -868,9 +832,48 @@ var bcrypt = require('bcrypt'), } }; + User.sendConfirmationEmail = function(email) { + if (meta.config['email:smtp:host'] && meta.config['email:smtp:port'] && meta.config['email:from']) { + var confirm_code = utils.generateUUID(), + confirm_link = nconf.get('url') + 'confirm/' + confirm_code, + confirm_email = global.templates['emails/header'] + global.templates['emails/email_confirm'].parse({ + 'CONFIRM_LINK': confirm_link + }) + global.templates['emails/footer'], + confirm_email_plaintext = global.templates['emails/email_confirm_plaintext'].parse({ + 'CONFIRM_LINK': confirm_link + }); + + // Email confirmation code + var expiry_time = Date.now() / 1000 + 60 * 60 * 2; + + db.setObjectField('email:confirm', email, confirm_code); + + db.setObjectField('confirm:email', confirm_code, email); + db.setObjectField('confirm:email', confirm_code + ':expire', expiry_time); + + // Send intro email w/ confirm code + var message = emailjs.message.create({ + text: confirm_email_plaintext, + from: meta.config['email:from'] || 'localhost@example.org', + to: email, + subject: '[NodeBB] Registration Email Verification', + attachment: [{ + data: confirm_email, + alternative: true + }] + }); + + emailjsServer.send(message, function(err, success) { + if (err) { + console.log(err); + } + }); + } + }; + User.email = { exists: function(socket, email, callback) { - User.getUidByEmail(email, function(exists) { + User.getUidByEmail(email, function(err, exists) { exists = !! exists; if (typeof callback !== 'function') { socket.emit('user.email.exists', { @@ -882,14 +885,30 @@ var bcrypt = require('bcrypt'), }); }, confirm: function(code, callback) { - RDB.get('confirm:' + code + ':email', function(err, email) { + db.getObjectFields('confirm:email', [code, code + ':expire'], function(err, data) { if (err) { - RDB.handle(err); + return callback({ + status:'error' + }); + } + + var email = data.email; + var expiry = data[code + ':expire']; + if (parseInt(expiry, 10) >= Date.now() / 1000) { + + db.deleteObjectField('confirm:email', code); + db.deleteObjectField('confirm:email', code + ':expire'); + + return callback({ + status: 'expired' + }); } if (email !== null) { - RDB.set('email:' + email + ':confirm', true); - RDB.del('confirm:' + code + ':email'); + db.setObjectField('email:confirm', email, true); + + db.deleteObjectField('confirm:email', code); + db.deleteObjectField('confirm:email', code + ':expire'); callback({ status: 'ok' }); @@ -908,7 +927,7 @@ var bcrypt = require('bcrypt'), async.parallel({ unread: function(next) { - RDB.zrevrange('uid:' + uid + ':notifications:unread', 0, 10, function(err, nids) { + db.getSortedSetRevRange('uid:' + uid + ':notifications:unread', 0, 10, function(err, nids) { // @todo handle err var unread = []; @@ -924,7 +943,7 @@ var bcrypt = require('bcrypt'), if (notif_data) { unread.push(notif_data); } else { - RDB.zrem('uid:' + uid + ':notifications:unread', nid); + db.sortedSetRemove('uid:' + uid + ':notifications:unread', nid); } next(); @@ -938,7 +957,7 @@ var bcrypt = require('bcrypt'), }); }, read: function(next) { - RDB.zrevrange('uid:' + uid + ':notifications:read', 0, 10, function(err, nids) { + db.getSortedSetRevRange('uid:' + uid + ':notifications:read', 0, 10, function(err, nids) { // @todo handle err var read = []; @@ -954,7 +973,7 @@ var bcrypt = require('bcrypt'), if (notif_data) { read.push(notif_data); } else { - RDB.zrem('uid:' + uid + ':notifications:read', nid); + db.sortedSetRemove('uid:' + uid + ':notifications:read', nid); } next(); @@ -986,6 +1005,7 @@ var bcrypt = require('bcrypt'), before = new Date(parseInt(before, 10)); } + // TODO : figure out how to do this with dbal RDB.multi() .zrevrangebyscore('uid:' + uid + ':notifications:read', before ? before.getTime(): now.getTime(), -Infinity, 'LIMIT', 0, limit) .zrevrangebyscore('uid:' + uid + ':notifications:unread', before ? before.getTime(): now.getTime(), -Infinity, 'LIMIT', 0, limit) @@ -1014,10 +1034,10 @@ var bcrypt = require('bcrypt'), }); }, getUnreadCount: function(uid, callback) { - RDB.zcount('uid:' + uid + ':notifications:unread', 0, 10, callback); + db.sortedSetCount('uid:' + uid + ':notifications:unread', 0, 10, callback); }, getUnreadByUniqueId: function(uid, uniqueId, callback) { - RDB.zrange('uid:' + uid + ':notifications:unread', 0, -1, function(err, nids) { + db.getSortedSetRange('uid:' + uid + ':notifications:unread', 0, -1, function(err, nids) { async.filter(nids, function(nid, next) { notifications.get(nid, uid, function(notifObj) { if (notifObj.uniqueId === uniqueId) { From 32d51182660ef47298d8eafac02bf343257c1f4a Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 21:20:55 -0500 Subject: [PATCH 09/83] tons more changes --- src/admin/user.js | 3 +- src/categories.js | 53 ++++++++++++++++--------------- src/database/redis.js | 73 +++++++++++++++++++++++++++++++++++++++++-- src/favourites.js | 21 ++++++------- src/feed.js | 6 ++-- src/login.js | 12 +++---- src/messaging.js | 29 ++++++++++------- src/notifications.js | 32 +++++++++---------- src/postTools.js | 14 ++++----- src/posts.js | 34 ++++++++++---------- src/routes/admin.js | 27 ++-------------- src/routes/user.js | 4 +-- src/threadTools.js | 50 ++++++++++++++++++----------- src/topics.js | 71 ++++++++++++++++++++--------------------- src/upgrade.js | 7 ++++- src/user.js | 2 +- src/websockets.js | 27 ++++++++++------ 17 files changed, 271 insertions(+), 194 deletions(-) diff --git a/src/admin/user.js b/src/admin/user.js index d9662749f1..0f37d8590e 100644 --- a/src/admin/user.js +++ b/src/admin/user.js @@ -1,5 +1,4 @@ -var RDB = require('../redis'), - utils = require('../../public/src/utils'), +var utils = require('../../public/src/utils'), user = require('../user'), groups = require('../groups'); diff --git a/src/categories.js b/src/categories.js index 6aff4bfbe7..31295405e0 100644 --- a/src/categories.js +++ b/src/categories.js @@ -1,4 +1,4 @@ -var RDB = require('./redis.js'), +var db = require('./database.js'), posts = require('./posts.js'), utils = require('./../public/src/utils.js'), user = require('./user.js'), @@ -12,13 +12,13 @@ var RDB = require('./redis.js'), "use strict"; Categories.create = function(data, callback) { - RDB.incr('global:next_category_id', function(err, cid) { + db.incrObjectField('global', 'nextCid', function(err, cid) { if (err) { return callback(err, null); } var slug = cid + '/' + utils.slugify(data.name); - RDB.rpush('categories:cid', cid); + db.listAppend('categories:cid', cid); var category = { cid: cid, @@ -33,7 +33,7 @@ var RDB = require('./redis.js'), order: data.order }; - RDB.hmset('category:' + cid, category, function(err, data) { + db.setObject('category:' + cid, category, function(err, data) { callback(err, category); }); }); @@ -134,15 +134,15 @@ var RDB = require('./redis.js'), }; Categories.getTopicIds = function(cid, start, stop, callback) { - RDB.zrevrange('categories:' + cid + ':tid', start, stop, callback); + db.getSortedSetRevRange('categories:' + cid + ':tid', start, stop, callback); }; Categories.getActiveUsers = function(cid, callback) { - RDB.smembers('cid:' + cid + ':active_users', callback); + db.getSetMembers('cid:' + cid + ':active_users', callback); }; Categories.getAllCategories = function(current_user, callback) { - RDB.lrange('categories:cid', 0, -1, function(err, cids) { + db.getListRange('categories:cid', 0, -1, function(err, cids) { if(err) { return callback(err); } @@ -155,7 +155,7 @@ var RDB = require('./redis.js'), }; Categories.getModerators = function(cid, callback) { - RDB.smembers('cid:' + cid + ':moderators', function(err, mods) { + db.getSetMembers('cid:' + cid + ':moderators', function(err, mods) { if (!err) { if (mods && mods.length) { user.getMultipleUserFields(mods, ['username'], function(err, moderators) { @@ -172,7 +172,7 @@ var RDB = require('./redis.js'), }; Categories.isTopicsRead = function(cid, uid, callback) { - RDB.zrange('categories:' + cid + ':tid', 0, -1, function(err, tids) { + db.getSortedSetRange('categories:' + cid + ':tid', 0, -1, function(err, tids) { topics.hasReadTopics(tids, uid, function(hasRead) { @@ -189,7 +189,7 @@ var RDB = require('./redis.js'), }; Categories.markAsRead = function(cid, uid) { - RDB.sadd('cid:' + cid + ':read_by_uid', uid); + db.setAdd('cid:' + cid + ':read_by_uid', uid); }; Categories.hasReadCategories = function(cids, uid, callback) { @@ -205,15 +205,17 @@ var RDB = require('./redis.js'), }; Categories.hasReadCategory = function(cid, uid, callback) { - RDB.sismember('cid:' + cid + ':read_by_uid', uid, function(err, hasRead) { - RDB.handle(err); + db.isSetMember('cid:' + cid + ':read_by_uid', uid, function(err, hasRead) { + if(err) { + return callback(false); + } callback(hasRead); }); }; Categories.getRecentReplies = function(cid, count, callback) { - RDB.zrevrange('categories:recent_posts:cid:' + cid, 0, (count < 10) ? 10 : count, function(err, pids) { + db.getSortedSetRevRange('categories:recent_posts:cid:' + cid, 0, (count < 10) ? 10 : count, function(err, pids) { if (err) { winston.err(err); @@ -242,8 +244,8 @@ var RDB = require('./redis.js'), return callback(err); } - RDB.zrem('categories:recent_posts:cid:' + oldCid, pid); - RDB.zadd('categories:recent_posts:cid:' + cid, timestamp, pid); + db.sortedSetRemove('categories:recent_posts:cid:' + oldCid, pid); + db.sortedSetAdd('categories:recent_posts:cid:' + cid, timestamp, pid); callback(null); }); } @@ -283,9 +285,9 @@ var RDB = require('./redis.js'), }; Categories.getCategoryData = function(cid, callback) { - RDB.exists('category:' + cid, function(err, exists) { + db.exists('category:' + cid, function(err, exists) { if (exists) { - RDB.hgetall('category:' + cid, callback); + db.getObject('category:' + cid, callback); } else { callback(new Error('No category found!')); } @@ -293,19 +295,19 @@ var RDB = require('./redis.js'), }; Categories.getCategoryField = function(cid, field, callback) { - RDB.hget('category:' + cid, field, callback); + db.getObjectField('category:' + cid, field, callback); }; Categories.getCategoryFields = function(cid, fields, callback) { - RDB.hmgetObject('category:' + cid, fields, callback); + db.getObjectFields('category:' + cid, fields, callback); }; Categories.setCategoryField = function(cid, field, value, callback) { - RDB.hset('category:' + cid, field, value, callback); + db.setObjectField('category:' + cid, field, value, callback); }; Categories.incrementCategoryFieldBy = function(cid, field, value, callback) { - RDB.hincrby('category:' + cid, field, value, callback); + db.incrObjectFieldBy('category:' + cid, field, value, callback); }; Categories.getCategories = function(cids, uid, callback) { @@ -349,7 +351,7 @@ var RDB = require('./redis.js'), Categories.isUserActiveIn = function(cid, uid, callback) { - RDB.lrange('uid:' + uid + ':posts', 0, -1, function(err, pids) { + db.getListRange('uid:' + uid + ':posts', 0, -1, function(err, pids) { if (err) { return callback(err, null); } @@ -387,12 +389,13 @@ var RDB = require('./redis.js'), }; Categories.addActiveUser = function(cid, uid) { - if(parseInt(uid, 10)) - RDB.sadd('cid:' + cid + ':active_users', uid); + if(parseInt(uid, 10)) { + db.setAdd('cid:' + cid + ':active_users', uid); + } }; Categories.removeActiveUser = function(cid, uid) { - RDB.srem('cid:' + cid + ':active_users', uid); + db.setRemove('cid:' + cid + ':active_users', uid); }; }(exports)); \ No newline at end of file diff --git a/src/database/redis.js b/src/database/redis.js index 9492470e0c..8000773a76 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -17,6 +17,9 @@ redisClient = redis.createClient(nconf.get('redis:port'), nconf.get('redis:host')); } + module.client = redisClient; + module.type = 'redis'; + if (nconf.get('redis:password')) { redisClient.auth(nconf.get('redis:password')); } @@ -68,6 +71,38 @@ }); } + + module.info = function(callback) { + redisClient.info(function (err, data) { + if(err) { + return callback(err); + } + + data = data.split("\r\n"); + var finalData = {}; + + for (var i in data) { + + if (data[i].indexOf(':') == -1 || !data[i]) + continue; + + try { + data[i] = data[i].replace(/:/, "\":\""); + var json = "{\"" + data[i] + "\"}"; + + var jsonObject = JSON.parse(json); + for (var key in jsonObject) { + finalData[key] = jsonObject[key]; + } + } catch (err) { + winston.warn('can\'t parse redis status variable, ignoring', i, data[i], err); + } + } + + callback(null, finalData); + }); + } + // key module.exists = function(key, callback) { @@ -80,6 +115,14 @@ redisClient.del(key, callback); } + module.get = function(key, callback) { + redisClient.get(key, callback); + } + + module.set = function(key, callback) { + redisClient.get(key, callback); + } + //hashes module.setObject = function(key, data, callback) { @@ -91,7 +134,7 @@ } module.getObject = function(key, callback) { - redisClient.hgetall(key, callback) + redisClient.hgetall(key, callback); } module.getObjectField = function(key, field, callback) { @@ -138,6 +181,10 @@ redisClient.hincrby(key, field, 1, callback); } + module.decrObjectField = function(key, field, callback) { + redisClient.hincrby(key, field, -1, callback); + } + module.incrObjectFieldBy = function(key, field, value, callback) { redisClient.hincrby(key, field, value, callback); } @@ -157,6 +204,16 @@ redisClient.sismember(key, value, callback); } + module.isMemberOfSets = function(sets, value, callback) { + var batch = redisClient.multi(); + + for (var i = 0, ii = sets.length; i < ii; i++) { + batch.sismember(sets[i], value); + } + + batch.exec(callback); + } + module.getSetMembers = function(key, callback) { redisClient.smembers(key, callback); } @@ -165,6 +222,10 @@ redisClient.scard(key, callback); } + module.setRemoveRandom = function(key, callback) { + redisClient.spop(key, callback); + } + // sorted sets module.sortedSetAdd = function(key, score, value, callback) { @@ -176,11 +237,15 @@ } module.getSortedSetRange = function(key, start, stop, callback) { - redisClient.zrange(set, start, stop, callback); + redisClient.zrange(key, start, stop, callback); } module.getSortedSetRevRange = function(key, start, stop, callback) { - redisClient.zrevrange(set, start, stop, callback); + redisClient.zrevrange(key, start, stop, callback); + } + + module.getSortedSetRevRangeByScore = function(args, callback) { + redisClient.zrevrangebyscore(args, callback); } module.sortedSetCount = function(key, min, max, callback) { @@ -200,5 +265,7 @@ redisClient.lrange(key, start, stop, callback); } + + }(exports)); diff --git a/src/favourites.js b/src/favourites.js index baaaa33122..36214e0d25 100644 --- a/src/favourites.js +++ b/src/favourites.js @@ -1,4 +1,4 @@ -var RDB = require('./redis'), +var db = require('./database'), posts = require('./posts'), user = require('./user'), websockets = require('./websockets') @@ -26,14 +26,14 @@ var RDB = require('./redis'), Favourites.hasFavourited(pid, uid, function (hasFavourited) { if (hasFavourited === 0) { - RDB.sadd('pid:' + pid + ':users_favourited', uid); - RDB.zadd('uid:' + uid + ':favourites', postData.timestamp, pid); + db.setAdd('pid:' + pid + ':users_favourited', uid); + db.sortedSetAdd('uid:' + uid + ':favourites', postData.timestamp, pid); - RDB.hincrby('post:' + pid, 'reputation', 1); + db.incrObjectFieldBy('post:' + pid, 'reputation', 1); if (uid !== postData.uid) { user.incrementUserFieldBy(postData.uid, 'reputation', 1, function (err, newreputation) { - RDB.zadd('users:reputation', newreputation, postData.uid); + db.sortedSetAdd('users:reputation', newreputation, postData.uid); }); } @@ -61,14 +61,14 @@ var RDB = require('./redis'), posts.getPostField(pid, 'uid', function (err, uid_of_poster) { Favourites.hasFavourited(pid, uid, function (hasFavourited) { if (hasFavourited === 1) { - RDB.srem('pid:' + pid + ':users_favourited', uid); - RDB.zrem('uid:' + uid + ':favourites', pid); + db.setRemove('pid:' + pid + ':users_favourited', uid); + db.sortedSetRemove('uid:' + uid + ':favourites', pid); - RDB.hincrby('post:' + pid, 'reputation', -1); + db.incrObjectFieldBy('post:' + pid, 'reputation', -1); if (uid !== uid_of_poster) { user.incrementUserFieldBy(uid_of_poster, 'reputation', -1, function (err, newreputation) { - RDB.zadd('users:reputation', newreputation, uid_of_poster); + db.sortedSetAdd('users:reputation', newreputation, uid_of_poster); }); } @@ -89,8 +89,7 @@ var RDB = require('./redis'), }; Favourites.hasFavourited = function (pid, uid, callback) { - RDB.sismember('pid:' + pid + ':users_favourited', uid, function (err, hasFavourited) { - RDB.handle(err); + db.isSetMember('pid:' + pid + ':users_favourited', uid, function (err, hasFavourited) { callback(hasFavourited); }); diff --git a/src/feed.js b/src/feed.js index 522edab086..fab4fe05a4 100644 --- a/src/feed.js +++ b/src/feed.js @@ -1,7 +1,7 @@ (function (Feed) { - var RDB = require('./redis.js'), - posts = require('./posts.js'), - topics = require('./topics.js'), + var db = require('./database'), + posts = require('./posts'), + topics = require('./topics'), categories = require('./categories'), fs = require('fs'), diff --git a/src/login.js b/src/login.js index d63355ad32..adce1f8bfc 100644 --- a/src/login.js +++ b/src/login.js @@ -1,9 +1,9 @@ -var user = require('./user.js'), +var user = require('./user'), bcrypt = require('bcrypt'), - RDB = require('./redis.js'), + db = require('./database'), path = require('path'), winston = require('winston'), - utils = require('./../public/src/utils.js'); + utils = require('./../public/src/utils'); (function(Login) { @@ -77,7 +77,7 @@ var user = require('./user.js'), // Save twitter-specific information to the user user.setUserField(uid, 'twid', twid); - RDB.hset('twid:uid', twid, uid); + db.setObjectField('twid:uid', twid, uid); // Save their photo, if present if (photos && photos.length > 0) { @@ -111,7 +111,7 @@ var user = require('./user.js'), var success = function(uid) { // Save google-specific information to the user user.setUserField(uid, 'gplusid', gplusid); - RDB.hset('gplusid:uid', gplusid, uid); + db.setObjectField('gplusid:uid', gplusid, uid); callback(null, { uid: uid }); @@ -154,7 +154,7 @@ var user = require('./user.js'), var success = function(uid) { // Save facebook-specific information to the user user.setUserField(uid, 'fbid', fbid); - RDB.hset('fbid:uid', fbid, uid); + db.setObjectField('fbid:uid', fbid, uid); callback(null, { uid: uid }); diff --git a/src/messaging.js b/src/messaging.js index 22e9abfeca..1dd891ffae 100644 --- a/src/messaging.js +++ b/src/messaging.js @@ -1,4 +1,4 @@ -var RDB = require('./redis'), +var db = require('./database'), async = require('async'), user = require('./user'); @@ -14,9 +14,10 @@ var RDB = require('./redis'), Messaging.addMessage = function(fromuid, touid, content, callback) { var uids = sortUids(fromuid, touid); - RDB.incr('global:next_message_id', function(err, mid) { - if (err) + db.incrObjectField('global', 'nextMid', function(err, mid) { + if (err) { return callback(err, null); + } var message = { content: content, @@ -25,8 +26,8 @@ var RDB = require('./redis'), touid: touid }; - RDB.hmset('message:' + mid, message); - RDB.rpush('messages:' + uids[0] + ':' + uids[1], mid); + db.setObject('message:' + mid, message); + db.listAppend('messages:' + uids[0] + ':' + uids[1], mid); callback(null, message); }); @@ -35,9 +36,10 @@ var RDB = require('./redis'), Messaging.getMessages = function(fromuid, touid, callback) { var uids = sortUids(fromuid, touid); - RDB.lrange('messages:' + uids[0] + ':' + uids[1], 0, -1, function(err, mids) { - if (err) + db.getListRange('messages:' + uids[0] + ':' + uids[1], 0, -1, function(err, mids) { + if (err) { return callback(err, null); + } if (!mids || !mids.length) { return callback(null, []); @@ -49,14 +51,16 @@ var RDB = require('./redis'), var messages = []; function getMessage(mid, next) { - RDB.hgetall('message:' + mid, function(err, message) { - if (err) + db.getObject('message:' + mid, function(err, message) { + if (err) { return next(err); + } - if (message.fromuid === fromuid) + if (message.fromuid === fromuid) { message.content = 'You : ' + message.content; - else + } else { message.content = tousername + ' : ' + message.content; + } messages.push(message); next(null); @@ -64,8 +68,9 @@ var RDB = require('./redis'), } async.eachSeries(mids, getMessage, function(err) { - if (err) + if (err) { return callback(err, null); + } callback(null, messages); }); diff --git a/src/notifications.js b/src/notifications.js index 06786119dd..a6f8fc64a9 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -1,9 +1,9 @@ -var RDB = require('./redis'), - async = require('async'), - utils = require('../public/src/utils'), +var async = require('async'), winston = require('winston'), cron = require('cron').CronJob, + db = require('./database'), + utils = require('../public/src/utils'), websockets = require('./websockets'); @@ -49,9 +49,9 @@ var RDB = require('./redis'), * (un)read list contains the same uniqueId, it will be removed, and * the new one put in its place. */ - RDB.incr('notifications:next_nid', function(err, nid) { - RDB.sadd('notifications', nid); - RDB.hmset('notifications:' + nid, { + db.incrObjectField('global', 'nextNid', function(err, nid) { + db.setAdd('notifications', nid); + db.setObject('notifications:' + nid, { text: text || '', path: path || null, datetime: Date.now(), @@ -91,7 +91,7 @@ var RDB = require('./redis'), if (parseInt(uids[x], 10) > 0) { (function(uid) { remove_by_uniqueId(notif_data.uniqueId, uid, function() { - RDB.zadd('uid:' + uid + ':notifications:unread', notif_data.datetime, nid); + db.sortedSetAdd('uid:' + uid + ':notifications:unread', notif_data.datetime, nid); websockets.in('uid_' + uid).emit('event:new_notification'); @@ -108,12 +108,12 @@ var RDB = require('./redis'), function remove_by_uniqueId(uniqueId, uid, callback) { async.parallel([ function(next) { - RDB.zrange('uid:' + uid + ':notifications:unread', 0, -1, function(err, nids) { + db.getSortedSetRange('uid:' + uid + ':notifications:unread', 0, -1, function(err, nids) { if (nids && nids.length > 0) { async.each(nids, function(nid, next) { Notifications.get(nid, uid, function(nid_info) { if (nid_info.uniqueId === uniqueId) { - RDB.zrem('uid:' + uid + ':notifications:unread', nid); + db.sortedSetRemove('uid:' + uid + ':notifications:unread', nid); } next(); @@ -127,12 +127,12 @@ var RDB = require('./redis'), }); }, function(next) { - RDB.zrange('uid:' + uid + ':notifications:read', 0, -1, function(err, nids) { + db.getSortedSetRange('uid:' + uid + ':notifications:read', 0, -1, function(err, nids) { if (nids && nids.length > 0) { async.each(nids, function(nid, next) { Notifications.get(nid, uid, function(nid_info) { if (nid_info && nid_info.uniqueId === uniqueId) { - RDB.zrem('uid:' + uid + ':notifications:read', nid); + db.sortedSetRemove('uid:' + uid + ':notifications:read', nid); } next(); @@ -155,8 +155,8 @@ var RDB = require('./redis'), Notifications.mark_read = function(nid, uid, callback) { if (parseInt(uid) > 0) { Notifications.get(nid, uid, function(notif_data) { - RDB.zrem('uid:' + uid + ':notifications:unread', nid); - RDB.zadd('uid:' + uid + ':notifications:read', notif_data.datetime, nid); + db.sortedSetRemove('uid:' + uid + ':notifications:unread', nid); + db.sortedSetAdd('uid:' + uid + ':notifications:read', notif_data.datetime, nid); if (callback) { callback(); } @@ -183,7 +183,7 @@ var RDB = require('./redis'), }; Notifications.mark_all_read = function(uid, callback) { - RDB.zrange('uid:' + uid + ':notifications:unread', 0, 10, function(err, nids) { + db.getSortedSetRange('uid:' + uid + ':notifications:unread', 0, 10, function(err, nids) { if (err) { return callback(err); } @@ -217,9 +217,9 @@ var RDB = require('./redis'), RDB.keys('uid:*:notifications:unread', next); }, "nids": function(next) { - RDB.smembers('notifications', function(err, nids) { + db.getSetMembers('notifications', function(err, nids) { async.filter(nids, function(nid, next) { - RDB.hget('notifications:' + nid, 'datetime', function(err, datetime) { + db.getObjectField('notifications:' + nid, 'datetime', function(err, datetime) { if (parseInt(datetime, 10) < cutoffTime) { next(true); } else { diff --git a/src/postTools.js b/src/postTools.js index c70e88f167..2641b8d9b2 100644 --- a/src/postTools.js +++ b/src/postTools.js @@ -1,4 +1,4 @@ -var RDB = require('./redis'), +var db = require('./database'), posts = require('./posts'), topics = require('./topics'), threadTools = require('./threadTools'), @@ -19,7 +19,7 @@ var RDB = require('./redis'), (function(PostTools) { PostTools.isMain = function(pid, tid, callback) { - RDB.lrange('tid:' + tid + ':posts', 0, 0, function(err, pids) { + db.getListRange('tid:' + tid + ':posts', 0, 0, function(err, pids) { if(err) { return callback(err); } @@ -131,14 +131,14 @@ var RDB = require('./redis'), PostTools.delete = function(uid, pid, callback) { var success = function() { posts.setPostField(pid, 'deleted', 1); - RDB.decr('totalpostcount'); + db.decrObjectField('global', 'postCount'); postSearch.remove(pid); posts.getPostFields(pid, ['tid', 'uid'], function(err, postData) { - RDB.hincrby('topic:' + postData.tid, 'postcount', -1); + db.incrObjectFieldBy('topic:' + postData.tid, 'postcount', -1); user.decrementUserFieldBy(postData.uid, 'postcount', 1, function(err, postcount) { - RDB.zadd('users:postcount', postcount, postData.uid); + db.sortedSetAdd('users:postcount', postcount, postData.uid); }); // Delete the thread if it is the last undeleted post @@ -180,10 +180,10 @@ var RDB = require('./redis'), PostTools.restore = function(uid, pid, callback) { var success = function() { posts.setPostField(pid, 'deleted', 0); - RDB.incr('totalpostcount'); + db.incrObjectField('global', 'postCount'); posts.getPostFields(pid, ['tid', 'uid', 'content'], function(err, postData) { - RDB.hincrby('topic:' + postData.tid, 'postcount', 1); + db.incrObjectFieldBy('topic:' + postData.tid, 'postcount', 1); user.incrementUserFieldBy(postData.uid, 'postcount', 1); diff --git a/src/posts.js b/src/posts.js index 646f38dded..49479428b6 100644 --- a/src/posts.js +++ b/src/posts.js @@ -1,4 +1,4 @@ -var RDB = require('./redis'), +var db = require('./database'), utils = require('./../public/src/utils'), user = require('./user'), topics = require('./topics'), @@ -34,7 +34,7 @@ var RDB = require('./redis'), callback(new Error('topic-locked'), null); } - RDB.incr('global:next_post_id', function(err, pid) { + db.incrObjectField('global', 'nextPid', function(err, pid) { if(err) { return callback(err, null); } @@ -57,7 +57,7 @@ var RDB = require('./redis'), 'deleted': 0 }; - RDB.hmset('post:' + pid, postData); + db.setObject('post:' + pid, postData); postData.favourited = false; postData.display_moderator_tools = true; @@ -67,26 +67,24 @@ var RDB = require('./redis'), topics.increasePostCount(tid); topics.updateTimestamp(tid, timestamp); - RDB.incr('totalpostcount'); + db.incrObjectField('global', 'postCount'); topics.getTopicFields(tid, ['cid', 'pinned'], function(err, topicData) { - RDB.handle(err); - var cid = topicData.cid; feed.updateTopic(tid); feed.updateRecent(); - RDB.zadd('categories:recent_posts:cid:' + cid, timestamp, pid); + db.sortedSetAdd('categories:recent_posts:cid:' + cid, timestamp, pid); if(topicData.pinned === '0') { - RDB.zadd('categories:' + cid + ':tid', timestamp, tid); + db.sortedSetAdd('categories:' + cid + ':tid', timestamp, tid); } - RDB.scard('cid:' + cid + ':active_users', function(err, amount) { + db.setCount('cid:' + cid + ':active_users', function(err, amount) { if (amount > 15) { - RDB.spop('cid:' + cid + ':active_users'); + db.setRemoveRandom('cid:' + cid + ':active_users'); } categories.addActiveUser(cid, uid); @@ -155,7 +153,7 @@ var RDB = require('./redis'), return next(err); } - RDB.del('cid:' + cid + ':read_by_uid'); + db.delete('cid:' + cid + ':read_by_uid'); next(); }); }, @@ -182,8 +180,10 @@ var RDB = require('./redis'), } Posts.getPostsByTid = function(tid, start, end, callback) { - RDB.lrange('tid:' + tid + ':posts', start, end, function(err, pids) { - RDB.handle(err); + db.getListRange('tid:' + tid + ':posts', start, end, function(err, pids) { + if(err) { + return callback(err); + } if (pids.length) { plugins.fireHook('filter:post.getTopic', pids, function(err, posts) { @@ -310,7 +310,7 @@ var RDB = require('./redis'), }; Posts.getPostData = function(pid, callback) { - RDB.hgetall('post:' + pid, function(err, data) { + db.getObject('post:' + pid, function(err, data) { if(err) { return callback(err, null); } @@ -325,7 +325,7 @@ var RDB = require('./redis'), } Posts.getPostFields = function(pid, fields, callback) { - RDB.hmgetObject('post:' + pid, fields, function(err, data) { + db.getObjectFields('post:' + pid, fields, function(err, data) { if(err) { return callback(err, null); } @@ -355,7 +355,7 @@ var RDB = require('./redis'), } Posts.setPostField = function(pid, field, value, callback) { - RDB.hset('post:' + pid, field, value, callback); + db.setObjectField('post:' + pid, field, value, callback); plugins.fireHook('action:post.setField', { 'pid': pid, 'field': field, @@ -499,7 +499,7 @@ var RDB = require('./redis'), } Posts.getFavourites = function(uid, callback) { - RDB.zrevrange('uid:' + uid + ':favourites', 0, -1, function(err, pids) { + db.getSortedSetRevRange('uid:' + uid + ':favourites', 0, -1, function(err, pids) { if (err) return callback(err, null); diff --git a/src/routes/admin.js b/src/routes/admin.js index 28dffa27d8..4e62b1a9a3 100644 --- a/src/routes/admin.js +++ b/src/routes/admin.js @@ -3,7 +3,7 @@ var nconf = require('nconf'), path = require('path'), winston = require('winston'), - RDB = require('./../redis'), + db = require('./../database'), user = require('./../user'), groups = require('../groups'), topics = require('./../topics'), @@ -267,29 +267,8 @@ var nconf = require('nconf'), app.namespace('/redis', function () { app.get('/', function (req, res) { - RDB.info(function (err, data) { - data = data.split("\r\n"); - var finalData = {}; - - for (var i in data) { - - if (data[i].indexOf(':') == -1 || !data[i]) - continue; - - try { - data[i] = data[i].replace(/:/, "\":\""); - var json = "{\"" + data[i] + "\"}"; - - var jsonObject = JSON.parse(json); - for (var key in jsonObject) { - finalData[key] = jsonObject[key]; - } - } catch (err) { - winston.warn('can\'t parse redis status variable, ignoring', i, data[i], err); - } - } - - res.json(finalData); + db.info(function (err, data) { + res.json(data); }); }); diff --git a/src/routes/user.js b/src/routes/user.js index b0de83ebcb..58231a16c6 100644 --- a/src/routes/user.js +++ b/src/routes/user.js @@ -9,7 +9,7 @@ var fs = require('fs'), postTools = require('../postTools'), utils = require('./../../public/src/utils'), meta = require('./../meta'), - RDB = require('./../redis'), + db = require('./../database'), websockets = require('./../websockets'); (function (User) { @@ -454,7 +454,7 @@ var fs = require('fs'), if(websockets.isUserOnline(user.uid)) { onlineUsers.push(user); } else { - RDB.zrem('users:online', user.uid); + db.sortedSetRemove('users:online', user.uid); } callback(null); } diff --git a/src/threadTools.js b/src/threadTools.js index a7df6eeaf8..331bc71298 100644 --- a/src/threadTools.js +++ b/src/threadTools.js @@ -1,4 +1,4 @@ -var RDB = require('./redis'), +var db = require('./database'), topics = require('./topics'), categories = require('./categories'), CategoryTools = require('./categoryTools'), @@ -17,8 +17,11 @@ var RDB = require('./redis'), (function(ThreadTools) { ThreadTools.exists = function(tid, callback) { - RDB.sismember('topics:tid', tid, function(err, ismember) { - if (err) RDB.handle(err); + db.isSetMember('topics:tid', tid, function(err, ismember) { + if (err) { + callback(false); + } + callback( !! ismember || false); }); } @@ -85,7 +88,7 @@ var RDB = require('./redis'), ThreadTools.delete = function(tid, callback) { topics.delete(tid); - RDB.decr('totaltopiccount'); + db.decrObjectField('global', 'topicCount'); ThreadTools.lock(tid); @@ -103,7 +106,7 @@ var RDB = require('./redis'), ThreadTools.restore = function(tid, socket, callback) { topics.restore(tid); - RDB.incr('totaltopiccount'); + db.incrObjectField('global', 'topicCount'); ThreadTools.unlock(tid); websockets.in('topic_' + tid).emit('event:topic_restored', { @@ -123,7 +126,7 @@ var RDB = require('./redis'), ThreadTools.pin = function(tid, socket) { topics.setTopicField(tid, 'pinned', 1); topics.getTopicField(tid, 'cid', function(err, cid) { - RDB.zadd('categories:' + cid + ':tid', Math.pow(2, 53), tid); + db.sortedSetAdd('categories:' + cid + ':tid', Math.pow(2, 53), tid); }); if (socket) { @@ -144,7 +147,7 @@ var RDB = require('./redis'), ThreadTools.unpin = function(tid, socket) { topics.setTopicField(tid, 'pinned', 0); topics.getTopicFields(tid, ['cid', 'lastposttime'], function(err, topicData) { - RDB.zadd('categories:' + topicData.cid + ':tid', topicData.lastposttime, tid); + db.sortedSetAdd('categories:' + topicData.cid + ':tid', topicData.lastposttime, tid); }); if (socket) { websockets.in('topic_' + tid).emit('event:topic_unpinned', { @@ -208,7 +211,7 @@ var RDB = require('./redis'), } ThreadTools.isFollowing = function(tid, current_user, callback) { - RDB.sismember('tid:' + tid + ':followers', current_user, function(err, following) { + db.isSetMember('tid:' + tid + ':followers', current_user, function(err, following) { callback(following); }); } @@ -216,7 +219,7 @@ var RDB = require('./redis'), ThreadTools.toggleFollow = function(tid, current_user, callback) { ThreadTools.isFollowing(tid, current_user, function(following) { if (!following) { - RDB.sadd('tid:' + tid + ':followers', current_user, function(err, success) { + db.setAdd('tid:' + tid + ':followers', current_user, function(err, success) { if (callback) { if (!err) { callback({ @@ -229,7 +232,7 @@ var RDB = require('./redis'), } }); } else { - RDB.srem('tid:' + tid + ':followers', current_user, function(err, success) { + db.setRemove('tid:' + tid + ':followers', current_user, function(err, success) { if (callback) { if (!err) { callback({ @@ -246,7 +249,7 @@ var RDB = require('./redis'), } ThreadTools.getFollowers = function(tid, callback) { - RDB.smembers('tid:' + tid + ':followers', function(err, followers) { + db.getSetMembers('tid:' + tid + ':followers', function(err, followers) { callback(err, followers.map(function(follower) { return parseInt(follower, 10); })); @@ -274,24 +277,33 @@ var RDB = require('./redis'), }); } ], function(err, results) { - if (!err) notifications.push(results[0], results[1]); - // Otherwise, do nothing + if (!err) { + notifications.push(results[0], results[1]); + } }); } ThreadTools.getLatestUndeletedPid = function(tid, callback) { - RDB.lrange('tid:' + tid + ':posts', 0, -1, function(err, pids) { - if (pids.length === 0) return callback(new Error('no-undeleted-pids-found')); + db.getListRange('tid:' + tid + ':posts', 0, -1, function(err, pids) { + if (pids.length === 0) { + return callback(new Error('no-undeleted-pids-found')); + } pids.reverse(); async.detectSeries(pids, function(pid, next) { posts.getPostField(pid, 'deleted', function(err, deleted) { - if (deleted === '0') next(true); - else next(false); + if (deleted === '0') { + next(true); + } else { + next(false); + } }); }, function(pid) { - if (pid) callback(null, pid); - else callback(new Error('no-undeleted-pids-found')); + if (pid) { + callback(null, pid); + } else { + callback(new Error('no-undeleted-pids-found')); + } }); }); } diff --git a/src/topics.js b/src/topics.js index 01b36c6136..1f33d5be51 100644 --- a/src/topics.js +++ b/src/topics.js @@ -5,7 +5,7 @@ var async = require('async'), reds = require('reds'), topicSearch = reds.createSearch('nodebbtopicsearch'), - RDB = require('./redis'), + db = require('./database'), posts = require('./posts'), utils = require('./../public/src/utils'), user = require('./user'), @@ -58,16 +58,16 @@ var async = require('async'), return callback(new Error('too-many-posts'), null); } - RDB.incr('next_topic_id', function(err, tid) { + db.incrObjectField('global', 'nextTid', function(err, tid) { if(err) { return callback(err); } - RDB.sadd('topics:tid', tid); + db.setAdd('topics:tid', tid); var slug = tid + '/' + utils.slugify(title); var timestamp = Date.now(); - RDB.hmset('topic:' + tid, { + db.setObject('topic:' + tid, { 'tid': tid, 'uid': uid, 'cid': cid, @@ -87,15 +87,15 @@ var async = require('async'), user.addTopicIdToUser(uid, tid); // let everyone know that there is an unread topic in this category - RDB.del('cid:' + cid + ':read_by_uid', function(err, data) { + db.delete('cid:' + cid + ':read_by_uid', function(err, data) { Topics.markAsRead(tid, uid); }); // in future it may be possible to add topics to several categories, so leaving the door open here. - RDB.zadd('categories:' + cid + ':tid', timestamp, tid); - RDB.hincrby('category:' + cid, 'topic_count', 1); - RDB.incr('totaltopiccount'); + db.sortedSetAdd('categories:' + cid + ':tid', timestamp, tid); + db.incrObjectField('category:' + cid, 'topic_count', 1); + db.incrObjectField('global', 'topicCount'); feed.updateCategory(cid); @@ -124,7 +124,7 @@ var async = require('async'), }; Topics.getTopicData = function(tid, callback) { - RDB.hgetall('topic:' + tid, function(err, data) { + db.getObject('topic:' + tid, function(err, data) { if(err) { return callback(err, null); } @@ -236,8 +236,7 @@ var async = require('async'), since = terms[term]; var args = ['topics:recent', '+inf', timestamp - since, 'LIMIT', start, end - start + 1]; - - RDB.zrevrangebyscore(args, function(err, tids) { + db.getSortedSetRevRangeByScore(args, function(err, tids) { if (err) { return callback(err); } @@ -272,7 +271,7 @@ var async = require('async'), return unreadTids.length < 21 && !done; }, function(callback) { - RDB.zrevrange('topics:recent', start, stop, function(err, tids) { + db.getSortedSetRevRange('topics:recent', start, stop, function(err, tids) { if (err) return callback(err); @@ -344,7 +343,7 @@ var async = require('async'), async.whilst( continueCondition, function(callback) { - RDB.zrevrange('topics:recent', start, stop, function(err, tids) { + db.getSortedSetRevRange('topics:recent', start, stop, function(err, tids) { if (err) return callback(err); @@ -597,7 +596,7 @@ var async = require('async'), } Topics.getAllTopics = function(limit, after, callback) { - RDB.smembers('topics:tid', function(err, tids) { + db.getSetMembers('topics:tid', function(err, tids) { if(err) { return callback(err, null); } @@ -643,7 +642,7 @@ var async = require('async'), } Topics.markAllRead = function(uid, callback) { - RDB.smembers('topics:tid', function(err, tids) { + db.getSetMembers('topics:tid', function(err, tids) { if (err) { return callback(err, null); } @@ -667,12 +666,12 @@ var async = require('async'), } Topics.markUnRead = function(tid, callback) { - RDB.del('tid:' + tid + ':read_by_uid', callback); + db.delete('tid:' + tid + ':read_by_uid', callback); } Topics.markAsRead = function(tid, uid) { - RDB.sadd('tid:' + tid + ':read_by_uid', uid); + db.setAdd('tid:' + tid + ':read_by_uid', uid); Topics.getTopicField(tid, 'cid', function(err, cid) { @@ -691,19 +690,19 @@ var async = require('async'), } Topics.hasReadTopics = function(tids, uid, callback) { - var batch = RDB.multi(); + var sets = []; for (var i = 0, ii = tids.length; i < ii; i++) { - batch.sismember('tid:' + tids[i] + ':read_by_uid', uid); + sets.push('tid:' + tids[i] + ':read_by_uid'); } - batch.exec(function(err, hasRead) { + db.isMemberOfSets(sets, uid, function(err, hasRead) { callback(hasRead); }); } Topics.hasReadTopic = function(tid, uid, callback) { - RDB.sismember('tid:' + tid + ':read_by_uid', uid, function(err, hasRead) { + db.isSetMember('tid:' + tid + ':read_by_uid', uid, function(err, hasRead) { if (err === null) { callback(hasRead); @@ -719,7 +718,9 @@ var async = require('async'), if (Array.isArray(tids)) { async.eachSeries(tids, function(tid, next) { Topics.getTeaser(tid, function(err, teaser_info) { - if (err) teaser_info = {}; + if (err) { + teaser_info = {}; + } teasers.push(teaser_info); next(); }); @@ -783,23 +784,23 @@ var async = require('async'), } Topics.getTopicField = function(tid, field, callback) { - RDB.hget('topic:' + tid, field, callback); + db.getObjectField('topic:' + tid, field, callback); } Topics.getTopicFields = function(tid, fields, callback) { - RDB.hmgetObject('topic:' + tid, fields, callback); + db.getObjectFields('topic:' + tid, fields, callback); } Topics.setTopicField = function(tid, field, value, callback) { - RDB.hset('topic:' + tid, field, value, callback); + db.setObjectField('topic:' + tid, field, value, callback); } Topics.increasePostCount = function(tid, callback) { - RDB.hincrby('topic:' + tid, 'postcount', 1, callback); + db.incrObjectField('topic:' + tid, 'postcount', callback); } Topics.increaseViewCount = function(tid, callback) { - RDB.hincrby('topic:' + tid, 'viewcount', 1, callback); + db.incrObjectField('topic:' + tid, 'viewcount', callback); } Topics.isLocked = function(tid, callback) { @@ -812,16 +813,16 @@ var async = require('async'), } Topics.updateTimestamp = function(tid, timestamp) { - RDB.zadd('topics:recent', timestamp, tid); + db.sortedSetAdd('topics:recent', timestamp, tid); Topics.setTopicField(tid, 'lastposttime', timestamp); } Topics.addPostToTopic = function(tid, pid) { - RDB.rpush('tid:' + tid + ':posts', pid); + db.listAppend('tid:' + tid + ':posts', pid); } Topics.getPids = function(tid, callback) { - RDB.lrange('tid:' + tid + ':posts', 0, -1, callback); + db.getListRange('tid:' + tid + ':posts', 0, -1, callback); } Topics.getUids = function(tid, callback) { @@ -848,23 +849,23 @@ var async = require('async'), Topics.delete = function(tid) { Topics.setTopicField(tid, 'deleted', 1); - RDB.zrem('topics:recent', tid); + db.sortedSetRemove('topics:recent', tid); Topics.getTopicField(tid, 'cid', function(err, cid) { feed.updateCategory(cid); - RDB.hincrby('category:' + cid, 'topic_count', -1); + db.incrObjectFieldBy('category:' + cid, 'topic_count', -1); }); } Topics.restore = function(tid) { Topics.setTopicField(tid, 'deleted', 0); Topics.getTopicField(tid, 'lastposttime', function(err, lastposttime) { - RDB.zadd('topics:recent', lastposttime, tid); + db.sortedSetAdd('topics:recent', lastposttime, tid); }); Topics.getTopicField(tid, 'cid', function(err, cid) { feed.updateCategory(cid); - RDB.hincrby('category:' + cid, 'topic_count', 1); + db.incrObjectFieldBy('category:' + cid, 'topic_count', 1); }); } @@ -885,7 +886,7 @@ var async = require('async'), } Topics.reIndexAll = function(callback) { - RDB.smembers('topics:tid', function(err, tids) { + db.getSetMembers('topics:tid', function(err, tids) { if (err) { callback(err, null); } else { diff --git a/src/upgrade.js b/src/upgrade.js index cec799914d..2fcac1b0b0 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -1,6 +1,11 @@ "use strict"; -var RDB = require('./redis.js'), +var db = require('./database'), + + // TODO: temp until upgrade is figured out with dbal, + // db.client is redisClient for now + RDB = db.client, + async = require('async'), winston = require('winston'), notifications = require('./notifications'), diff --git a/src/user.js b/src/user.js index d932d235f5..0c227052c3 100644 --- a/src/user.js +++ b/src/user.js @@ -486,7 +486,7 @@ var bcrypt = require('bcrypt'), User.unfollow = function(uid, unfollowid, callback) { db.setRemove('following:' + uid, unfollowid, function(err, data) { if (!err) { - RDB.srem('followers:' + unfollowid, uid, function(err, data) { + db.setRemove('followers:' + unfollowid, uid, function(err, data) { callback(data); }); } else { diff --git a/src/websockets.js b/src/websockets.js index 87eba11946..f5ec7a2628 100644 --- a/src/websockets.js +++ b/src/websockets.js @@ -10,9 +10,13 @@ var cookie = require('cookie'), winston = require('winston'), RedisStoreLib = require('connect-redis')(express), - RDB = require('./redis'), + db = require('./database'), + + redis = require('redis'), + redisClient = redis.createClient(nconf.get('redis:port'), nconf.get('redis:host')), + RedisStore = new RedisStoreLib({ - client: RDB, + client: redisClient, ttl: 60 * 60 * 24 * 14 }), @@ -71,8 +75,11 @@ websockets.init = function(io) { socketCookieParser(hs, {}, function(err) { sessionID = socket.handshake.signedCookies["express.sid"]; RedisStore.get(sessionID, function(err, sessionData) { - if (!err && sessionData && sessionData.passport && sessionData.passport.user) uid = users[sessionID] = sessionData.passport.user; - else uid = users[sessionID] = 0; + if (!err && sessionData && sessionData.passport && sessionData.passport.user) { + uid = users[sessionID] = sessionData.passport.user; + } else { + uid = users[sessionID] = 0; + } userSockets[uid] = userSockets[uid] || []; userSockets[uid].push(socket); @@ -89,7 +96,7 @@ websockets.init = function(io) { if (uid) { - RDB.zadd('users:online', Date.now(), uid, function(err, data) { + db.sortedSetAdd('users:online', Date.now(), uid, function(err, data) { socket.join('uid_' + uid); user.getUserField(uid, 'username', function(err, username) { @@ -119,7 +126,7 @@ websockets.init = function(io) { delete users[sessionID]; delete userSockets[uid]; if (uid) { - RDB.zrem('users:online', uid, function(err, data) { + db.sortedSetRemove('users:online', uid, function(err, data) { }); } } @@ -1108,14 +1115,14 @@ websockets.init = function(io) { function emitTopicPostStats() { - RDB.mget(['totaltopiccount', 'totalpostcount'], function(err, data) { + db.getObjectFields('global', ['topicCount', 'postCount'], function(err, data) { if (err) { return winston.err(err); } var stats = { - topics: data[0] ? data[0] : 0, - posts: data[1] ? data[1] : 0 + topics: data.topicCount ? data.topicCount : 0, + posts: data.postCount ? data.postCount : 0 }; io.sockets.emit('post.stats', stats); @@ -1123,7 +1130,7 @@ websockets.init = function(io) { } websockets.emitUserCount = function() { - RDB.get('usercount', function(err, count) { + db.getObjectField('global', 'userCount', function(err, count) { io.sockets.emit('user.count', { count: count }); From fc066c21bf33a32131b72fb889462c083b5d13a8 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 21:33:35 -0500 Subject: [PATCH 10/83] added upgrade for global keys --- src/upgrade.js | 56 +++++++++++++++++++++++++++++++++++++++++++++++--- src/user.js | 4 ++-- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/upgrade.js b/src/upgrade.js index 2fcac1b0b0..3ba15a97e2 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -16,7 +16,7 @@ var db = require('./database'), Upgrade.check = function(callback) { // IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema - var latestSchema = new Date(2013, 10, 26).getTime(); + var latestSchema = new Date(2013, 11, 2).getTime(); RDB.get('schemaDate', function(err, value) { if (parseInt(value, 10) >= latestSchema) { @@ -184,7 +184,7 @@ Upgrade.upgrade = function(callback) { }, function(next) { thisSchemaDate = new Date(2013, 10, 26).getTime(); - if (schemaDate < thisSchemaDate || 1) { + if (schemaDate < thisSchemaDate) { categories.getAllCategories(0, function(err, categories) { function updateIcon(category, next) { @@ -214,7 +214,57 @@ Upgrade.upgrade = function(callback) { winston.info('[2013/11/26] Update to Category icons skipped.'); next(); } - } + }, + function(next) { + + function updateKeyToHash(key, next) { + RDB.get(key, function(err, value) { + RDB.hset('global', newKeys[key], value, next); + }); + } + + thisSchemaDate = new Date(2013, 11, 2).getTime(); + if (schemaDate < thisSchemaDate) { + + var keys = [ + 'global:next_user_id', + 'next_topic_id', + 'next_gid', + 'notifications:next_nid', + 'global:next_category_id', + 'global:next_message_id', + 'global:next_post_id', + 'usercount', + 'totaltopiccount', + 'totalpostcount' + ]; + + var newKeys = { + 'global:next_user_id':'nextUid', + 'next_topic_id':'nextTid', + 'next_gid':'nextGid', + 'notifications:next_nid':'nextNid', + 'global:next_category_id':'nextCid', + 'global:next_message_id':'nextMid', + 'global:next_post_id':'nextPid', + 'usercount':'userCount', + 'totaltopiccount':'topicCount', + 'totalpostcount':'postCount' + }; + + async.each(keys, updateKeyToHash, function(err) { + if(err) { + return next(err); + } + winston.info('[2013/12/2] Updated global keys to hash.'); + next(); + }); + + } else { + winston.info('[2013/12/2] Update to global keys skipped'); + next(); + } + }, // Add new schema updates here // IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema IN LINE 12!!! ], function(err) { diff --git a/src/user.js b/src/user.js index 0c227052c3..3c05b4f49a 100644 --- a/src/user.js +++ b/src/user.js @@ -107,7 +107,7 @@ var bcrypt = require('bcrypt'), } plugins.fireHook('action:user.create', {uid: uid, username: username, email: email, picture: gravatar, timestamp: timestamp}); - db.incrObjectField('global', 'usercount'); + db.incrObjectField('global', 'userCount'); db.sortedSetAdd('users:joindate', timestamp, uid); db.sortedSetAdd('users:postcount', 0, uid); @@ -597,7 +597,7 @@ var bcrypt = require('bcrypt'), }; User.count = function(socket) { - db.getObjectField('global', 'usercount', function(err, count) { + db.getObjectField('global', 'userCount', function(err, count) { if(err) { return; } From b900bc9cce1ccbf5e46a4075651271e7399e871d Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 21:58:37 -0500 Subject: [PATCH 11/83] more fixes --- src/database/redis.js | 7 ++++++- src/login.js | 1 + src/routes/authentication.js | 10 ++++++++-- src/user.js | 4 ++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/database/redis.js b/src/database/redis.js index 8000773a76..e6f7ef8e13 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -126,7 +126,12 @@ //hashes module.setObject = function(key, data, callback) { - redisClient.hmset(key, data, callback); + // TODO: this crashes if callback isnt supplied -baris + redisClient.hmset(key, data, function(err, res) { + if(callback) { + callback(err, res); + } + }); } module.setObjectField = function(key, field, value, callback) { diff --git a/src/login.js b/src/login.js index adce1f8bfc..d20fec6c30 100644 --- a/src/login.js +++ b/src/login.js @@ -43,6 +43,7 @@ var user = require('./user'), } if (res) { + console.log('logged in', uid); next(null, { user: { uid: uid diff --git a/src/routes/authentication.js b/src/routes/authentication.js index 15951ece8b..9f9a5f0155 100644 --- a/src/routes/authentication.js +++ b/src/routes/authentication.js @@ -14,8 +14,12 @@ passport.use(new passportLocal(function(user, password, next) { login_module.loginViaLocal(user, password, function(err, login) { - if (!err) next(null, login.user); - else next(null, false, err); + if (!err) { + console.log('LOGGED IN'); + next(null, login.user); + } else { + next(null, false, err); + } }); })); @@ -189,7 +193,9 @@ }); app.post('/register', function(req, res) { + console.log('CALLING USER CREATE'); user.create(req.body.username, req.body.password, req.body.email, function(err, uid) { + console.log('USER CREATE DONE', err, uid); if (err === null && uid) { req.login({ uid: uid diff --git a/src/user.js b/src/user.js index 3c05b4f49a..00557d2e31 100644 --- a/src/user.js +++ b/src/user.js @@ -68,7 +68,7 @@ var bcrypt = require('bcrypt'), return callback(err); } - db.incrObjectField('global', 'nextUserId', function(err, uid) { + db.incrObjectField('global', 'nextUid', function(err, uid) { if(err) { return callback(err); } @@ -103,7 +103,7 @@ var bcrypt = require('bcrypt'), if (email !== undefined) { db.setObjectField('email:uid', email, uid); - User.sendConfirmationEmail(email); + //User.sendConfirmationEmail(email); } plugins.fireHook('action:user.create', {uid: uid, username: username, email: email, picture: gravatar, timestamp: timestamp}); From 95db5f93cb1de9ee85a9846af8bde81830b42a0a Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 22:01:29 -0500 Subject: [PATCH 12/83] remvoed console.logs --- src/routes/authentication.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/routes/authentication.js b/src/routes/authentication.js index 9f9a5f0155..14c3fb5e98 100644 --- a/src/routes/authentication.js +++ b/src/routes/authentication.js @@ -193,9 +193,7 @@ }); app.post('/register', function(req, res) { - console.log('CALLING USER CREATE'); user.create(req.body.username, req.body.password, req.body.email, function(err, uid) { - console.log('USER CREATE DONE', err, uid); if (err === null && uid) { req.login({ uid: uid From 6d79521922b72f0d09a5a2228e9713363052c44b Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 22:33:55 -0500 Subject: [PATCH 13/83] moved connect-redis to the redis.db file, expose the sessionStore from redis.js db file, do the same for mongo db with mongo-connect --- src/database/redis.js | 8 ++++++++ src/login.js | 1 - src/routes/authentication.js | 1 - src/webserver.js | 16 ++-------------- src/websockets.js | 10 +--------- 5 files changed, 11 insertions(+), 25 deletions(-) diff --git a/src/database/redis.js b/src/database/redis.js index e6f7ef8e13..462bd46eb5 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -6,6 +6,9 @@ redis = require('redis'), winston = require('winston'), nconf = require('nconf'), + express = require('express'), + connectRedis = require('connect-redis')(express), + redis_socket_or_host = nconf.get('redis:host'), utils = require('./../../public/src/utils.js'); @@ -17,6 +20,11 @@ redisClient = redis.createClient(nconf.get('redis:port'), nconf.get('redis:host')); } + module.sessionStore = new connectRedis({ + client: redisClient, + ttl: 60 * 60 * 24 * 30 + }); + module.client = redisClient; module.type = 'redis'; diff --git a/src/login.js b/src/login.js index d20fec6c30..adce1f8bfc 100644 --- a/src/login.js +++ b/src/login.js @@ -43,7 +43,6 @@ var user = require('./user'), } if (res) { - console.log('logged in', uid); next(null, { user: { uid: uid diff --git a/src/routes/authentication.js b/src/routes/authentication.js index 14c3fb5e98..ba2aae842d 100644 --- a/src/routes/authentication.js +++ b/src/routes/authentication.js @@ -15,7 +15,6 @@ passport.use(new passportLocal(function(user, password, next) { login_module.loginViaLocal(user, password, function(err, login) { if (!err) { - console.log('LOGGED IN'); next(null, login.user); } else { next(null, false, err); diff --git a/src/webserver.js b/src/webserver.js index 60c5e87e62..58bd187ae7 100644 --- a/src/webserver.js +++ b/src/webserver.js @@ -5,7 +5,6 @@ var path = require('path'), express_namespace = require('express-namespace'), WebServer = express(), server = require('http').createServer(WebServer), - RedisStore = require('connect-redis')(express), nconf = require('nconf'), winston = require('winston'), validator = require('validator'), @@ -142,19 +141,8 @@ var path = require('path'), app.use(express.cookieParser()); // If you want to parse cookies (res.cookies) // TODO : this uses redis - /*app.use(express.session({ - store: new RedisStore({ - client: RDB, - ttl: 60 * 60 * 24 * 30 - }), - secret: nconf.get('secret'), - key: 'express.sid', - cookie: { - maxAge: 60 * 60 * 24 * 30 * 1000 // 30 days - } - }));*/ - - app.use(express.cookieSession({ + app.use(express.session({ + store: db.sessionStore, secret: nconf.get('secret'), key: 'express.sid', cookie: { diff --git a/src/websockets.js b/src/websockets.js index f5ec7a2628..19324e695e 100644 --- a/src/websockets.js +++ b/src/websockets.js @@ -12,14 +12,6 @@ var cookie = require('cookie'), RedisStoreLib = require('connect-redis')(express), db = require('./database'), - redis = require('redis'), - redisClient = redis.createClient(nconf.get('redis:port'), nconf.get('redis:host')), - - RedisStore = new RedisStoreLib({ - client: redisClient, - ttl: 60 * 60 * 24 * 14 - }), - user = require('./user'), Groups = require('./groups'), posts = require('./posts'), @@ -74,7 +66,7 @@ websockets.init = function(io) { // Validate the session, if present socketCookieParser(hs, {}, function(err) { sessionID = socket.handshake.signedCookies["express.sid"]; - RedisStore.get(sessionID, function(err, sessionData) { + db.sessionStore.get(sessionID, function(err, sessionData) { if (!err && sessionData && sessionData.passport && sessionData.passport.user) { uid = users[sessionID] = sessionData.passport.user; } else { From d4eddc6e2c89f5c7766d6da1eda344f5f2136570 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 22:36:26 -0500 Subject: [PATCH 14/83] lcased groups --- src/websockets.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/websockets.js b/src/websockets.js index 19324e695e..6a8d998c09 100644 --- a/src/websockets.js +++ b/src/websockets.js @@ -13,7 +13,7 @@ var cookie = require('cookie'), db = require('./database'), user = require('./user'), - Groups = require('./groups'), + groups = require('./groups'), posts = require('./posts'), favourites = require('./favourites'), utils = require('../public/src/utils'), @@ -1008,16 +1008,16 @@ websockets.init = function(io) { }; if (set) { - Groups.joinByGroupName('cid:' + cid + ':privileges:' + privilege, uid, cb); + groups.joinByGroupName('cid:' + cid + ':privileges:' + privilege, uid, cb); } else { - Groups.leaveByGroupName('cid:' + cid + ':privileges:' + privilege, uid, cb); + groups.leaveByGroupName('cid:' + cid + ':privileges:' + privilege, uid, cb); } }); socket.on('api:admin.categories.getPrivilegeSettings', function(cid, callback) { async.parallel({ "+r": function(next) { - Groups.getByGroupName('cid:' + cid + ':privileges:+r', { expand: true }, function(err, groupObj) { + groups.getByGroupName('cid:' + cid + ':privileges:+r', { expand: true }, function(err, groupObj) { if (!err) { next.apply(this, arguments); } else { @@ -1028,7 +1028,7 @@ websockets.init = function(io) { }); }, "+w": function(next) { - Groups.getByGroupName('cid:' + cid + ':privileges:+w', { expand: true }, function(err, groupObj) { + groups.getByGroupName('cid:' + cid + ':privileges:+w', { expand: true }, function(err, groupObj) { if (!err) { next.apply(this, arguments); } else { @@ -1069,19 +1069,19 @@ websockets.init = function(io) { */ socket.on('api:groups.create', function(data, callback) { - Groups.create(data.name, data.description, function(err, groupObj) { + groups.create(data.name, data.description, function(err, groupObj) { callback(err ? err.message : null, groupObj || undefined); }); }); socket.on('api:groups.delete', function(gid, callback) { - Groups.destroy(gid, function(err) { + groups.destroy(gid, function(err) { callback(err ? err.message : null, err ? null : 'OK'); }); }); socket.on('api:groups.get', function(gid, callback) { - Groups.get(gid, { + groups.get(gid, { expand: true }, function(err, groupObj) { callback(err ? err.message : null, groupObj || undefined); @@ -1089,15 +1089,15 @@ websockets.init = function(io) { }); socket.on('api:groups.join', function(data, callback) { - Groups.join(data.gid, data.uid, callback); + groups.join(data.gid, data.uid, callback); }); socket.on('api:groups.leave', function(data, callback) { - Groups.leave(data.gid, data.uid, callback); + groups.leave(data.gid, data.uid, callback); }); socket.on('api:groups.update', function(data, callback) { - Groups.update(data.gid, data.values, function(err) { + groups.update(data.gid, data.values, function(err) { callback(err ? err.message : null); }); }); From 304285e8748cb25f83a141c822d851e541917661 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 22:48:32 -0500 Subject: [PATCH 15/83] some mongo stuff --- src/database/mongo.js | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index e9f495264b..56eda239ca 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -2,31 +2,35 @@ (function(module) { 'use strict'; - var mongoClient, - mongo = require('mongo') + var mongodb = require('mongodb') + mongoClient = mongodb.MongoClient, winston = require('winston'), nconf = require('nconf'), - mongoHost = nconf.get('mongo:host'), - utils = require('./../../public/src/utils.js'); + express = require('express'), + mongoStore = require('connect-mongo')(express); + mongoHost = nconf.get('mongo:host'); - // temp, look this up - mongoClient = mongo.createClient(nconf.get('mongo:port'), nconf.get('mongo:host')); + + // mongoClient.connect("mongodb://localhost:27017/exampleDb", function(err, db) { + mongoClient.connect('mongodb://' + mongoHost + ':' + nconf.get('mongo:port') + '/' + nconf.get('mongo:database'), function(err, db) { + if(err) { + winston.error("NodeBB could not connect to your Mongo database. Mongo returned the following error: " + error.message); + process.exit(); + } + }); // look up how its done in mongo /*if (nconf.get('mongo:password')) { redisClient.auth(nconf.get('mongo:password')); } + */ + + // TODO: fill out settings.db + module.sessionStore = new mongoStore({ + db: settings.db + }); - var db = parseInt(nconf.get('mongo:database'), 10); - if (db){ - mongoClient.select(db, function(error) { - if(error) { - winston.error("NodeBB could not connect to your Redis database. Redis returned the following error: " + error.message); - process.exit(); - } - }); - }*/ // // Exported functions From 1f52717f1e459d4ff51b46f8ecc1c3c592d62d6a Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Mon, 2 Dec 2013 22:50:39 -0500 Subject: [PATCH 16/83] fixed indent --- src/database/mongo.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 56eda239ca..2864d91395 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -27,8 +27,8 @@ // TODO: fill out settings.db module.sessionStore = new mongoStore({ - db: settings.db - }); + db: settings.db + }); From 91d6f83de47916e913dda95fcb042a95b2d52f00 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Tue, 3 Dec 2013 13:36:44 -0500 Subject: [PATCH 17/83] more mongo work --- app.js | 5 +- package.json | 2 + src/database/mongo.js | 184 +++++++++++-- src/plugins.js | 588 ++++++++++++++++++++++-------------------- src/user.js | 2 +- 5 files changed, 470 insertions(+), 311 deletions(-) diff --git a/app.js b/app.js index b1e27a6c9b..1c9e30e979 100644 --- a/app.js +++ b/app.js @@ -89,14 +89,15 @@ webserver = require('./src/webserver'), SocketIO = require('socket.io').listen(global.server, { log: false, transports: ['websocket', 'xhr-polling', 'jsonp-polling', 'flashsocket'], 'browser client minification': true}), websockets = require('./src/websockets'), - plugins = require('./src/plugins'), // Don't remove this - plugins initializes itself + plugins = require('./src/plugins'), notifications = require('./src/notifications'), upgrade = require('./src/upgrade'); upgrade.check(function(schema_ok) { if (schema_ok || nconf.get('check-schema') === false) { websockets.init(SocketIO); - + console.log('calling plugins init'); + plugins.init(); global.templates = {}; global.translator = translator; diff --git a/package.json b/package.json index 4710b9daa0..af257f5775 100644 --- a/package.json +++ b/package.json @@ -15,11 +15,13 @@ "dependencies": { "socket.io": "~0.9.16", "redis": "0.8.3", + "mongodb": "1.3.20", "express": "3.2.0", "express-namespace": "~0.1.1", "emailjs": "0.3.4", "cookie": "0.0.6", "connect-redis": "1.4.5", + "connect-mongo": "0.4.0", "passport": "0.1.17", "passport-local": "0.1.6", "passport-twitter": "0.1.5", diff --git a/src/database/mongo.js b/src/database/mongo.js index 2864d91395..6d3b4f6e5c 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -2,74 +2,212 @@ (function(module) { 'use strict'; - var mongodb = require('mongodb') - mongoClient = mongodb.MongoClient, + var Db = require('mongodb').Db, + mongoClient = require('mongodb').MongoClient, + Server = require('mongodb').Server, winston = require('winston'), nconf = require('nconf'), express = require('express'), - mongoStore = require('connect-mongo')(express); - mongoHost = nconf.get('mongo:host'); + mongoStore = require('connect-mongo')(express), + mongoHost = nconf.get('mongo:host'), + db; - // mongoClient.connect("mongodb://localhost:27017/exampleDb", function(err, db) { - mongoClient.connect('mongodb://' + mongoHost + ':' + nconf.get('mongo:port') + '/' + nconf.get('mongo:database'), function(err, db) { + var db = new Db(nconf.get('mongo:database'), new Server(mongoHost, nconf.get('mongo:port')), {w:1}); + //console.log(db.collection); + + db.open(function(err, _db) { + //mongoClient.connect('mongodb://'+ mongoHost + ':' + nconf.get('mongo:port') + '/' + nconf.get('mongo:database'), function(err, _db) { + console.log('WE ARE CONNECTED'); if(err) { - winston.error("NodeBB could not connect to your Mongo database. Mongo returned the following error: " + error.message); + winston.error("NodeBB could not connect to your Mongo database. Mongo returned the following error: " + err.message); process.exit(); } + + // TODO: fill out settings.db + module.sessionStore = new mongoStore({ + db: db + }); + + db.collection('objects').findOne({_key:'config'}, {timeout:true}, function(err, item) { + console.log('fail'); + console.log(item); + callback(err, item); + }); + + }); + + db.createCollection('objects', function(err, collection) { + console.log('collection created', err, collection); + }); + + db.createCollection('sets', function(err, collection) { + }); + // look up how its done in mongo /*if (nconf.get('mongo:password')) { redisClient.auth(nconf.get('mongo:password')); } */ - // TODO: fill out settings.db - module.sessionStore = new mongoStore({ - db: settings.db - }); - - // // Exported functions // module.getFileName = function(callback) { - // TODO : get mongodb filename + throw new Error('not-implemented'); + } + + module.info = function(callback) { + throw new Error('not-implemented'); + } + + // key + + module.exists = function(key, callback) { + throw new Error('not-implemented'); + } + + module.delete = function(key, callback) { + throw new Error('not-implemented'); + } + + module.get = function(key, callback) { + throw new Error('not-implemented'); + } + + module.set = function(key, callback) { + throw new Error('not-implemented'); } + //hashes module.setObject = function(key, data, callback) { - // TODO : implement in mongo + data['_key'] = key; + db.collection('objects').insert(data, {w:1}, function(err, result) { + callback(err, result); + }); } - module.setObjectField = function(key, field, callback) { - // TODO : implement in mongo + module.setObjectField = function(key, field, value, callback) { + db.collection('objects').update(); } module.getObject = function(key, callback) { - // TODO : implement in mongo + console.log('calling findOne'); + db.collection('objects').findOne({_key:key}, {timeout:true},function(err, item) { + console.log(item); + callback(err, item); + }); } module.getObjectField = function(key, field, callback) { - // TODO : implement in mongo + throw new Error('not-implemented'); } module.getObjectFields = function(key, fields, callback) { - // TODO : implement in mongo + throw new Error('not-implemented'); + } + + module.getObjectValues = function(key, callback) { + throw new Error('not-implemented'); + } + + module.isObjectField = function(key, field, callback) { + throw new Error('not-implemented'); } module.deleteObjectField = function(key, field, callback) { - // TODO : implement in mongo + throw new Error('not-implemented'); + } + + module.incrObjectField = function(key, field, callback) { + throw new Error('not-implemented'); + } + + module.decrObjectField = function(key, field, callback) { + throw new Error('not-implemented'); } - module.incrObjectField = function(key, field, value, callback) { - // TODO : implement in mongo + module.incrObjectFieldBy = function(key, field, value, callback) { + throw new Error('not-implemented'); } + // sets + module.setAdd = function(key, value, callback) { + throw new Error('not-implemented'); + } + + module.setRemove = function(key, value, callback) { + throw new Error('not-implemented'); + } + + module.isSetMember = function(key, value, callback) { + throw new Error('not-implemented'); + } + + module.isMemberOfSets = function(sets, value, callback) { + throw new Error('not-implemented'); + } + + module.getSetMembers = function(key, callback) { + console.log('getting set members'); + db.collection('sets').findOne({_key:key}, function(err, data) { + console.log('derp', err, data); + callback(err, data); + }); + } + + module.setCount = function(key, callback) { + throw new Error('not-implemented'); + } + + module.setRemoveRandom = function(key, callback) { + throw new Error('not-implemented'); + } + + // sorted sets + + module.sortedSetAdd = function(key, score, value, callback) { + throw new Error('not-implemented'); + } + + module.sortedSetRemove = function(key, value, callback) { + throw new Error('not-implemented'); + } + + module.getSortedSetRange = function(key, start, stop, callback) { + throw new Error('not-implemented'); + } + + module.getSortedSetRevRange = function(key, start, stop, callback) { + throw new Error('not-implemented'); + } + + module.getSortedSetRevRangeByScore = function(args, callback) { + throw new Error('not-implemented'); + } + + module.sortedSetCount = function(key, min, max, callback) { + throw new Error('not-implemented'); + } + + // lists + module.listPrepend = function(key, value, callback) { + throw new Error('not-implemented'); + } + + module.listAppend = function(key, value, callback) { + throw new Error('not-implemented'); + } + + module.getListRange = function(key, start, stop, callback) { + throw new Error('not-implemented'); + } }(exports)); diff --git a/src/plugins.js b/src/plugins.js index 7dbc0a6bdd..38f4db2d93 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -3,325 +3,343 @@ var fs = require('fs'), async = require('async'), winston = require('winston'), eventEmitter = require('events').EventEmitter, - db = require('./database'), + db = require('./database'); - plugins = { - libraries: {}, - loadedHooks: {}, - staticDirs: {}, - cssFiles: [], +(function(Plugins) { - // Events - readyEvent: new eventEmitter, + Plugins.libraries = {}; + Plugins.loadedHooks = {}; + Plugins.staticDirs = {}; + Plugins.cssFiles = []; - init: function() { - if (this.initialized) return; - if (global.env === 'development') winston.info('[plugins] Initializing plugins system'); + Plugins.initialized = false; - this.reload(function(err) { - if (err) { - if (global.env === 'development') winston.info('[plugins] NodeBB encountered a problem while loading plugins', err.message); - return; + // Events + Plugins.readyEvent = new eventEmitter; + + Plugins.init = function() { + console.log('plugins init called'); + if (Plugins.initialized) { + return; + } + + if (global.env === 'development') { + winston.info('[plugins] Initializing plugins system'); + } + + Plugins.reload(function(err) { + if (err) { + if (global.env === 'development') { + winston.info('[plugins] NodeBB encountered a problem while loading plugins', err.message); } + return; + } - if (global.env === 'development') winston.info('[plugins] Plugins OK'); + if (global.env === 'development') { + winston.info('[plugins] Plugins OK'); + } + Plugins.initialized = true; + plugins.readyEvent.emit('ready'); + }); + }; + + Plugins.ready = function(callback) { + if (!Plugins.initialized) { + Plugins.readyEvent.once('ready', callback); + } else { + callback(); + } + }; + + Plugins.reload = function(callback) { + // Resetting all local plugin data + Plugins.loadedHooks = {}; + Plugins.staticDirs = {}; + Plugins.cssFiles.length = 0; + + // Read the list of activated plugins and require their libraries + async.waterfall([ + function(next) { + db.getSetMembers('plugins:active', next); + }, + function(plugins, next) { + if (plugins && Array.isArray(plugins) && plugins.length > 0) { + async.each(plugins, function(plugin, next) { + var modulePath = path.join(__dirname, '../node_modules/', plugin); + if (fs.existsSync(modulePath)) { + Plugins.loadPlugin(modulePath, next); + } else { + if (global.env === 'development') { + winston.warn('[plugins] Plugin \'' + plugin + '\' not found'); + } + next(); // Ignore this plugin silently + } + }, next); + } else next(); + }, + function(next) { + if (global.env === 'development') winston.info('[plugins] Sorting hooks to fire in priority sequence'); + Object.keys(Plugins.loadedHooks).forEach(function(hook) { + var hooks = Plugins.loadedHooks[hook]; + hooks = hooks.sort(function(a, b) { + return a.priority - b.priority; + }); + }); - plugins.initialized = true; - plugins.readyEvent.emit('ready'); - }); - }, - ready: function(callback) { - if (!this.initialized) this.readyEvent.once('ready', callback); - else callback(); - }, - initialized: false, - reload: function(callback) { - var _self = this; - - // Resetting all local plugin data - this.loadedHooks = {}; - this.staticDirs = {}; - this.cssFiles.length = 0; - - // Read the list of activated plugins and require their libraries - async.waterfall([ + next(); + } + ], callback); + }; + + Plugins.loadPlugin = function(pluginPath, callback) { + fs.readFile(path.join(pluginPath, 'plugin.json'), function(err, data) { + if (err) { + return callback(err); + } + + var pluginData = JSON.parse(data), + libraryPath, staticDir; + + async.parallel([ function(next) { - db.getSetMembers('plugins:active', next); - }, - function(plugins, next) { - if (plugins && Array.isArray(plugins) && plugins.length > 0) { - async.each(plugins, function(plugin, next) { - var modulePath = path.join(__dirname, '../node_modules/', plugin); - if (fs.existsSync(modulePath)) _self.loadPlugin(modulePath, next); - else { - if (global.env === 'development') winston.warn('[plugins] Plugin \'' + plugin + '\' not found'); - next(); // Ignore this plugin silently + if (pluginData.library) { + libraryPath = path.join(pluginPath, pluginData.library); + + fs.exists(libraryPath, function(exists) { + if (exists) { + if (!Plugins.libraries[pluginData.id]) { + Plugins.libraries[pluginData.id] = require(libraryPath); + } + + // Register hooks for this plugin + if (pluginData.hooks && Array.isArray(pluginData.hooks) && pluginData.hooks.length > 0) { + async.each(pluginData.hooks, function(hook, next) { + Plugins.registerHook(pluginData.id, hook, next); + }, next); + } else { + next(null); + } + } else { + winston.warn('[plugins.reload] Library not found for plugin: ' + pluginData.id); + next(); } - }, next); - } else next(); + }); + } else { + winston.warn('[plugins.reload] Library not found for plugin: ' + pluginData.id); + next(); + } }, function(next) { - if (global.env === 'development') winston.info('[plugins] Sorting hooks to fire in priority sequence'); - Object.keys(_self.loadedHooks).forEach(function(hook) { - var hooks = _self.loadedHooks[hook]; - hooks = hooks.sort(function(a, b) { - return a.priority - b.priority; + // Static Directories for Plugins + if (pluginData.staticDir) { + staticDir = path.join(pluginPath, pluginData.staticDir); + + fs.exists(staticDir, function(exists) { + if (exists) { + Plugins.staticDirs[pluginData.id] = staticDir; + next(); + } else next(); }); - }); - - next(); - } - ], callback); - }, - loadPlugin: function(pluginPath, callback) { - var _self = this; - - fs.readFile(path.join(pluginPath, 'plugin.json'), function(err, data) { - if (err) return callback(err); - - var pluginData = JSON.parse(data), - libraryPath, staticDir; - - async.parallel([ - function(next) { - if (pluginData.library) { - libraryPath = path.join(pluginPath, pluginData.library); - - fs.exists(libraryPath, function(exists) { - if (exists) { - if (!_self.libraries[pluginData.id]) { - _self.libraries[pluginData.id] = require(libraryPath); - } - - // Register hooks for this plugin - if (pluginData.hooks && Array.isArray(pluginData.hooks) && pluginData.hooks.length > 0) { - async.each(pluginData.hooks, function(hook, next) { - _self.registerHook(pluginData.id, hook, next); - }, next); - } else { - next(null); - } - } else { - winston.warn('[plugins.reload] Library not found for plugin: ' + pluginData.id); - next(); - } - }); - } else { - winston.warn('[plugins.reload] Library not found for plugin: ' + pluginData.id); - next(); + } else next(); + }, + function(next) { + // CSS Files for plugins + if (pluginData.css && pluginData.css instanceof Array) { + if (global.env === 'development') { + winston.info('[plugins] Found ' + pluginData.css.length + ' CSS file(s) for plugin ' + pluginData.id); } - }, - function(next) { - // Static Directories for Plugins - if (pluginData.staticDir) { - staticDir = path.join(pluginPath, pluginData.staticDir); - - fs.exists(staticDir, function(exists) { - if (exists) { - _self.staticDirs[pluginData.id] = staticDir; - next(); - } else next(); - }); - } else next(); - }, - function(next) { - // CSS Files for plugins - if (pluginData.css && pluginData.css instanceof Array) { - if (global.env === 'development') { - winston.info('[plugins] Found ' + pluginData.css.length + ' CSS file(s) for plugin ' + pluginData.id); - } - _self.cssFiles = _self.cssFiles.concat(pluginData.css.map(function(file) { - return path.join('/plugins', pluginData.id, file); - })); + Plugins.cssFiles = Plugins.cssFiles.concat(pluginData.css.map(function(file) { + return path.join('/plugins', pluginData.id, file); + })); - next(); - } else next(); - } - ], function(err) { - if (!err) { - if (global.env === 'development') winston.info('[plugins] Loaded plugin: ' + pluginData.id); - callback(); - } else callback(new Error('Could not load plugin system')) - }); + next(); + } else next(); + } + ], function(err) { + if (!err) { + if (global.env === 'development') winston.info('[plugins] Loaded plugin: ' + pluginData.id); + callback(); + } else callback(new Error('Could not load plugin system')) }); - }, - registerHook: function(id, data, callback) { - /* - `data` is an object consisting of (* is required): - `data.hook`*, the name of the NodeBB hook - `data.method`*, the method called in that plugin - `data.callbacked`, whether or not the hook expects a callback (true), or a return (false). Only used for filters. (Default: false) - `data.priority`, the relative priority of the method when it is eventually called (default: 10) - */ - var _self = this; - - if (data.hook && data.method) { - data.id = id; - if (!data.priority) data.priority = 10; - data.method = data.method.split('.').reduce(function(memo, prop) { - if (memo[prop]) { - return memo[prop]; - } else { - // Couldn't find method by path, assuming property with periods in it (evil!) - _self.libraries[data.id][data.method]; - } - }, _self.libraries[data.id]); - - _self.loadedHooks[data.hook] = _self.loadedHooks[data.hook] || []; - _self.loadedHooks[data.hook].push(data); - - if (global.env === 'development') winston.info('[plugins] Hook registered: ' + data.hook + ' will call ' + id); - callback(); - } else return; - }, - fireHook: function(hook, args, callback) { - var _self = this - hookList = this.loadedHooks[hook]; - - if (hookList && Array.isArray(hookList)) { - //if (global.env === 'development') winston.info('[plugins] Firing hook: \'' + hook + '\''); - var hookType = hook.split(':')[0]; - switch (hookType) { - case 'filter': - async.reduce(hookList, args, function(value, hookObj, next) { - if (hookObj.method) { - if (hookObj.callbacked) { // If a callback is present (asynchronous method) - hookObj.method.call(_self.libraries[hookObj.id], value, next); - } else { // Synchronous method - value = hookObj.method.call(_self.libraries[hookObj.id], value); - next(null, value); - } - } else { - if (global.env === 'development') winston.info('[plugins] Expected method \'' + hookObj.method + '\' in plugin \'' + hookObj.id + '\' not found, skipping.'); + }); + }; + + Plugins.registerHook = function(id, data, callback) { + /* + `data` is an object consisting of (* is required): + `data.hook`*, the name of the NodeBB hook + `data.method`*, the method called in that plugin + `data.callbacked`, whether or not the hook expects a callback (true), or a return (false). Only used for filters. (Default: false) + `data.priority`, the relative priority of the method when it is eventually called (default: 10) + */ + + if (data.hook && data.method) { + data.id = id; + if (!data.priority) data.priority = 10; + data.method = data.method.split('.').reduce(function(memo, prop) { + if (memo[prop]) { + return memo[prop]; + } else { + // Couldn't find method by path, assuming property with periods in it (evil!) + Plugins.libraries[data.id][data.method]; + } + }, Plugins.libraries[data.id]); + + Plugins.loadedHooks[data.hook] = Plugins.loadedHooks[data.hook] || []; + Plugins.loadedHooks[data.hook].push(data); + + if (global.env === 'development') { + winston.info('[plugins] Hook registered: ' + data.hook + ' will call ' + id); + } + callback(); + } else return; + }; + + Plugins.fireHook = function(hook, args, callback) { + hookList = Plugins.loadedHooks[hook]; + + if (hookList && Array.isArray(hookList)) { + //if (global.env === 'development') winston.info('[plugins] Firing hook: \'' + hook + '\''); + var hookType = hook.split(':')[0]; + switch (hookType) { + case 'filter': + async.reduce(hookList, args, function(value, hookObj, next) { + if (hookObj.method) { + if (hookObj.callbacked) { // If a callback is present (asynchronous method) + hookObj.method.call(Plugins.libraries[hookObj.id], value, next); + } else { // Synchronous method + value = hookObj.method.call(Plugins.libraries[hookObj.id], value); next(null, value); } - }, function(err, value) { - if (err) { - if (global.env === 'development') { - winston.info('[plugins] Problem executing hook: ' + hook); - } + } else { + if (global.env === 'development') winston.info('[plugins] Expected method \'' + hookObj.method + '\' in plugin \'' + hookObj.id + '\' not found, skipping.'); + next(null, value); + } + }, function(err, value) { + if (err) { + if (global.env === 'development') { + winston.info('[plugins] Problem executing hook: ' + hook); } + } - callback.apply(plugins, arguments); - }); - break; - case 'action': - async.each(hookList, function(hookObj) { - if (hookObj.method) { - hookObj.method.call(_self.libraries[hookObj.id], args); - } else { - if (global.env === 'development') { - winston.info('[plugins] Expected method \'' + hookObj.method + '\' in plugin \'' + hookObj.id + '\' not found, skipping.'); - } + callback.apply(plugins, arguments); + }); + break; + case 'action': + async.each(hookList, function(hookObj) { + if (hookObj.method) { + hookObj.method.call(Plugins.libraries[hookObj.id], args); + } else { + if (global.env === 'development') { + winston.info('[plugins] Expected method \'' + hookObj.method + '\' in plugin \'' + hookObj.id + '\' not found, skipping.'); } - }); - break; - default: - // Do nothing... - break; - } - } else { - // Otherwise, this hook contains no methods - var returnVal = args; - if (callback) { - callback(null, returnVal); - } + } + }); + break; + default: + // Do nothing... + break; + } + } else { + // Otherwise, this hook contains no methods + var returnVal = args; + if (callback) { + callback(null, returnVal); } - }, - isActive: function(id, callback) { - db.isSetMember('plugins:active', id, callback); - }, - toggleActive: function(id, callback) { - this.isActive(id, function(err, active) { + } + }; + + Plugins.isActive = function(id, callback) { + db.isSetMember('plugins:active', id, callback); + }; + + Plugins.toggleActive = function(id, callback) { + Plugins.isActive(id, function(err, active) { + if (err) { + if (global.env === 'development') winston.info('[plugins] Could not toggle active state on plugin \'' + id + '\''); + return; + } + + db[(active ? 'setRemove' : 'setAdd')]('plugins:active', id, function(err, success) { if (err) { if (global.env === 'development') winston.info('[plugins] Could not toggle active state on plugin \'' + id + '\''); return; } - db[(active ? 'setRemove' : 'setAdd')]('plugins:active', id, function(err, success) { - if (err) { - if (global.env === 'development') winston.info('[plugins] Could not toggle active state on plugin \'' + id + '\''); - return; - } + // Reload meta data + plugins.reload(function() { + // (De)activation Hooks + plugins.fireHook('action:plugin.' + (active ? 'de' : '') + 'activate', id); - // Reload meta data - plugins.reload(function() { - // (De)activation Hooks - plugins.fireHook('action:plugin.' + (active ? 'de' : '') + 'activate', id); - - if (callback) { - callback({ - id: id, - active: !active - }); - } - }); + if (callback) { + callback({ + id: id, + active: !active + }); + } }); }); - }, - showInstalled: function(callback) { - var _self = this; - npmPluginPath = path.join(__dirname, '../node_modules'); - - async.waterfall([ - function(next) { - fs.readdir(npmPluginPath, function(err, dirs) { - dirs = dirs.map(function(file) { - return path.join(npmPluginPath, file); - }).filter(function(file) { - var stats = fs.statSync(file); - if (stats.isDirectory() && file.substr(npmPluginPath.length + 1, 14) === 'nodebb-plugin-') return true; - else return false; - }); + }); + } - next(err, dirs); + Plugins.showInstalled = function(callback) { + npmPluginPath = path.join(__dirname, '../node_modules'); + + async.waterfall([ + function(next) { + fs.readdir(npmPluginPath, function(err, dirs) { + dirs = dirs.map(function(file) { + return path.join(npmPluginPath, file); + }).filter(function(file) { + var stats = fs.statSync(file); + if (stats.isDirectory() && file.substr(npmPluginPath.length + 1, 14) === 'nodebb-plugin-') return true; + else return false; }); - }, - function(files, next) { - var plugins = []; - - async.each(files, function(file, next) { - var configPath; - - async.waterfall([ - function(next) { - fs.readFile(path.join(file, 'plugin.json'), next); - }, - function(configJSON, next) { - try { - var config = JSON.parse(configJSON); - } catch (err) { - winston.warn("Plugin: " + file + " is corrupted or invalid. Please check plugin.json for errors.") - return next(err, null); - } - - _self.isActive(config.id, function(err, active) { - if (err) next(new Error('no-active-state')); - delete config.library; - delete config.hooks; - config.active = active; - config.activeText = ' ' + (active ? 'Dea' : 'A') + 'ctivate'; - next(null, config); - }); + next(err, dirs); + }); + }, + function(files, next) { + var plugins = []; + + async.each(files, function(file, next) { + var configPath; + + async.waterfall([ + function(next) { + fs.readFile(path.join(file, 'plugin.json'), next); + }, + function(configJSON, next) { + try { + var config = JSON.parse(configJSON); + } catch (err) { + winston.warn("Plugin: " + file + " is corrupted or invalid. Please check plugin.json for errors.") + return next(err, null); } - ], function(err, config) { - if (err) return next(); // Silently fail - plugins.push(config); - next(); - }); - }, function(err) { - next(null, plugins); - }); - } - ], function(err, plugins) { - callback(err, plugins); - }); - } - } + Plugins.isActive(config.id, function(err, active) { + if (err) next(new Error('no-active-state')); -plugins.init(); + delete config.library; + delete config.hooks; + config.active = active; + config.activeText = ' ' + (active ? 'Dea' : 'A') + 'ctivate'; + next(null, config); + }); + } + ], function(err, config) { + if (err) return next(); // Silently fail -module.exports = plugins; \ No newline at end of file + plugins.push(config); + next(); + }); + }, function(err) { + next(null, plugins); + }); + } + ], function(err, plugins) { + callback(err, plugins); + }); + } +}(exports)); diff --git a/src/user.js b/src/user.js index 00557d2e31..686b8e39c3 100644 --- a/src/user.js +++ b/src/user.js @@ -103,7 +103,7 @@ var bcrypt = require('bcrypt'), if (email !== undefined) { db.setObjectField('email:uid', email, uid); - //User.sendConfirmationEmail(email); + User.sendConfirmationEmail(email); } plugins.fireHook('action:user.create', {uid: uid, username: username, email: email, picture: gravatar, timestamp: timestamp}); From e862a1c4cc7fd1f511f64d9c9e094ad962994301 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Tue, 3 Dec 2013 14:21:08 -0500 Subject: [PATCH 18/83] added init method to database, progress made --- app.js | 112 +++++++++++++++++++++--------------------- src/database/mongo.js | 93 +++++++++++++++++++++-------------- src/database/redis.js | 11 +++-- src/plugins.js | 3 +- src/upgrade.js | 5 +- 5 files changed, 123 insertions(+), 101 deletions(-) diff --git a/app.js b/app.js index 1c9e30e979..9c839a94d0 100644 --- a/app.js +++ b/app.js @@ -60,7 +60,7 @@ nconf.file({ file: __dirname + '/config.json' }); - meta = require('./src/meta.js'); + meta = require('./src/meta'); nconf.set('url', nconf.get('base_url') + (nconf.get('use_port') ? ':' + nconf.get('port') : '') + nconf.get('relative_path') + '/'); nconf.set('upload_url', nconf.get('url') + 'uploads/'); @@ -73,60 +73,62 @@ winston.info('Base Configuration OK.'); } - meta.configs.init(function () { - - // - // TODO : figure out reds search after dbal is complete - // - //var reds = require('reds'), - // db = require('./src/database'); - /*reds.createClient = function () { - return reds.client || (reds.client = db); - };*/ - - var templates = require('./public/src/templates'), - translator = require('./public/src/translator'), - webserver = require('./src/webserver'), - SocketIO = require('socket.io').listen(global.server, { log: false, transports: ['websocket', 'xhr-polling', 'jsonp-polling', 'flashsocket'], 'browser client minification': true}), - websockets = require('./src/websockets'), - plugins = require('./src/plugins'), - notifications = require('./src/notifications'), - upgrade = require('./src/upgrade'); - - upgrade.check(function(schema_ok) { - if (schema_ok || nconf.get('check-schema') === false) { - websockets.init(SocketIO); - console.log('calling plugins init'); - plugins.init(); - global.templates = {}; - global.translator = translator; - - translator.loadServer(); - - var customTemplates = meta.config['theme:templates'] ? path.join(__dirname, 'node_modules', meta.config['theme:id'], meta.config['theme:templates']) : false; - - // todo: replace below with read directory code, derp. - templates.init([ - 'header', 'footer', 'logout', 'outgoing', 'admin/header', 'admin/footer', 'admin/index', - 'emails/reset', 'emails/reset_plaintext', 'emails/email_confirm', 'emails/email_confirm_plaintext', - 'emails/header', 'emails/footer', - - 'noscript/header', 'noscript/home', 'noscript/category', 'noscript/topic' - ], customTemplates); - - - plugins.ready(function() { - templates.ready(webserver.init); - }); - - notifications.init(); - } else { - winston.warn('Your NodeBB schema is out-of-date. Please run the following command to bring your dataset up to spec:'); - winston.warn(' node app --upgrade'); - winston.warn('To ignore this error (not recommended):'); - winston.warn(' node app --no-check-schema') - process.exit(); - } + require('./src/database').init(function(err) { + meta.configs.init(function () { + + // + // TODO : figure out reds search after dbal is complete + // + //var reds = require('reds'), + // db = require('./src/database'); + /*reds.createClient = function () { + return reds.client || (reds.client = db); + };*/ + + var templates = require('./public/src/templates'), + translator = require('./public/src/translator'), + webserver = require('./src/webserver'), + SocketIO = require('socket.io').listen(global.server, { log: false, transports: ['websocket', 'xhr-polling', 'jsonp-polling', 'flashsocket'], 'browser client minification': true}), + websockets = require('./src/websockets'), + plugins = require('./src/plugins'), + notifications = require('./src/notifications'), + upgrade = require('./src/upgrade'); + + upgrade.check(function(schema_ok) { + if (schema_ok || nconf.get('check-schema') === false) { + websockets.init(SocketIO); + + plugins.init(); + global.templates = {}; + global.translator = translator; + + translator.loadServer(); + + var customTemplates = meta.config['theme:templates'] ? path.join(__dirname, 'node_modules', meta.config['theme:id'], meta.config['theme:templates']) : false; + + // todo: replace below with read directory code, derp. + templates.init([ + 'header', 'footer', 'logout', 'outgoing', 'admin/header', 'admin/footer', 'admin/index', + 'emails/reset', 'emails/reset_plaintext', 'emails/email_confirm', 'emails/email_confirm_plaintext', + 'emails/header', 'emails/footer', + + 'noscript/header', 'noscript/home', 'noscript/category', 'noscript/topic' + ], customTemplates); + + + plugins.ready(function() { + templates.ready(webserver.init); + }); + + notifications.init(); + } else { + winston.warn('Your NodeBB schema is out-of-date. Please run the following command to bring your dataset up to spec:'); + winston.warn(' node app --upgrade'); + winston.warn('To ignore this error (not recommended):'); + winston.warn(' node app --no-check-schema') + process.exit(); + } + }); }); }); } else if (nconf.get('setup') || nconf.get('install') || !fs.existsSync(__dirname + '/config.json')) { diff --git a/src/database/mongo.js b/src/database/mongo.js index 6d3b4f6e5c..801abb6dd7 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -2,9 +2,7 @@ (function(module) { 'use strict'; - var Db = require('mongodb').Db, - mongoClient = require('mongodb').MongoClient, - Server = require('mongodb').Server, + var mongoClient = require('mongodb').MongoClient, winston = require('winston'), nconf = require('nconf'), express = require('express'), @@ -12,45 +10,36 @@ mongoHost = nconf.get('mongo:host'), db; + module.init = function(callback) { + mongoClient.connect('mongodb://'+ mongoHost + ':' + nconf.get('mongo:port') + '/' + nconf.get('mongo:database'), function(err, _db) { + db = _db; + console.log('WE ARE CONNECTED'); - var db = new Db(nconf.get('mongo:database'), new Server(mongoHost, nconf.get('mongo:port')), {w:1}); - //console.log(db.collection); + if(err) { + winston.error("NodeBB could not connect to your Mongo database. Mongo returned the following error: " + err.message); + process.exit(); + } - db.open(function(err, _db) { - //mongoClient.connect('mongodb://'+ mongoHost + ':' + nconf.get('mongo:port') + '/' + nconf.get('mongo:database'), function(err, _db) { - console.log('WE ARE CONNECTED'); - if(err) { - winston.error("NodeBB could not connect to your Mongo database. Mongo returned the following error: " + err.message); - process.exit(); - } - - // TODO: fill out settings.db - module.sessionStore = new mongoStore({ - db: db - }); - - db.collection('objects').findOne({_key:'config'}, {timeout:true}, function(err, item) { - console.log('fail'); - console.log(item); - callback(err, item); - }); - - }); + // TODO: fill out settings.db + module.sessionStore = new mongoStore({ + db: db + }); - db.createCollection('objects', function(err, collection) { - console.log('collection created', err, collection); - }); - db.createCollection('sets', function(err, collection) { + db.createCollection('objects', function(err, collection) { + }); - }); + db.createCollection('sets', function(err, collection) { + }); - - // look up how its done in mongo - /*if (nconf.get('mongo:password')) { - redisClient.auth(nconf.get('mongo:password')); + callback(err); + }); + // look up how its done in mongo + /*if (nconf.get('mongo:password')) { + redisClient.auth(nconf.get('mongo:password')); + } + */ } - */ // @@ -97,18 +86,46 @@ module.getObject = function(key, callback) { console.log('calling findOne'); - db.collection('objects').findOne({_key:key}, {timeout:true},function(err, item) { + db.collection('objects').findOne({_key:key}, function(err, item) { console.log(item); callback(err, item); }); } module.getObjectField = function(key, field, callback) { - throw new Error('not-implemented'); + module.getObjectFields(key, [field], function(err, data) { + if(err) { + return callback(err); + } + + callback(null, data[field]); + }) } module.getObjectFields = function(key, fields, callback) { - throw new Error('not-implemented'); + + var _fields = {}; + for(var i=0; i=0) { /* If redis.host contains a path name character, use the unix dom sock connection. ie, /tmp/redis.sock */ redisClient = redis.createClient(nconf.get('redis:host')); @@ -20,14 +21,13 @@ redisClient = redis.createClient(nconf.get('redis:port'), nconf.get('redis:host')); } + module.client = redisClient; + module.sessionStore = new connectRedis({ client: redisClient, ttl: 60 * 60 * 24 * 30 }); - module.client = redisClient; - module.type = 'redis'; - if (nconf.get('redis:password')) { redisClient.auth(nconf.get('redis:password')); } @@ -43,6 +43,11 @@ }); } + module.init = function(callback) { + callback(null); + } + + /* * A possibly more efficient way of doing multiple sismember calls */ diff --git a/src/plugins.js b/src/plugins.js index 38f4db2d93..0ba6eae81d 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -18,7 +18,6 @@ var fs = require('fs'), Plugins.readyEvent = new eventEmitter; Plugins.init = function() { - console.log('plugins init called'); if (Plugins.initialized) { return; } @@ -39,7 +38,7 @@ var fs = require('fs'), winston.info('[plugins] Plugins OK'); } Plugins.initialized = true; - plugins.readyEvent.emit('ready'); + Plugins.readyEvent.emit('ready'); }); }; diff --git a/src/upgrade.js b/src/upgrade.js index 3ba15a97e2..49bedc2a2c 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -1,10 +1,9 @@ "use strict"; -var db = require('./database'), +var //db = require('./database'), // TODO: temp until upgrade is figured out with dbal, - // db.client is redisClient for now - RDB = db.client, + RDB = require('./database/redis').client, async = require('async'), winston = require('winston'), From 10474f8e2a75d4bc4fcc511bcb6bf271abdf7b0f Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Tue, 3 Dec 2013 15:17:42 -0500 Subject: [PATCH 19/83] more mongo work --- src/database/mongo.js | 33 ++++++++++++++++++------ src/routes/debug.js | 60 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 801abb6dd7..d0768ccba7 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -32,6 +32,7 @@ db.createCollection('sets', function(err, collection) { }); + callback(err); }); // look up how its done in mongo @@ -74,20 +75,27 @@ //hashes module.setObject = function(key, data, callback) { + console.log('SET OBJECT CALLED', key, data); data['_key'] = key; - db.collection('objects').insert(data, {w:1}, function(err, result) { + db.collection('objects').update({_key:key}, {$set:data}, {upsert:true, w: 1}, function(err, result) { + console.log('SET OBJECT COMPLETE', err, result); callback(err, result); }); } module.setObjectField = function(key, field, value, callback) { - db.collection('objects').update(); + var data = {}; + data[field] = value; + db.collection('objects').update({_key:key}, {$set:data}, {upsert:true, w: 1}, function(err, result) { + console.log('SET OBJECT COMPLETE', err, result); + callback(err, result); + }); } module.getObject = function(key, callback) { - console.log('calling findOne'); + console.log('GET OBJECT', key); db.collection('objects').findOne({_key:key}, function(err, item) { - console.log(item); + console.log('RETURNING OBJECT', item); callback(err, item); }); } @@ -106,7 +114,7 @@ var _fields = {}; for(var i=0; i Date: Tue, 3 Dec 2013 16:50:06 -0500 Subject: [PATCH 20/83] implemented getObjectValues in mongo --- src/database/mongo.js | 29 +++++++++++++++++++++-------- src/routes/debug.js | 10 +++++++++- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index d0768ccba7..d26f33a6ea 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -72,13 +72,15 @@ throw new Error('not-implemented'); } + module.keys = function(key, callback) { + throw new Error('not-implemented'); + } + //hashes module.setObject = function(key, data, callback) { - console.log('SET OBJECT CALLED', key, data); data['_key'] = key; db.collection('objects').update({_key:key}, {$set:data}, {upsert:true, w: 1}, function(err, result) { - console.log('SET OBJECT COMPLETE', err, result); callback(err, result); }); } @@ -87,15 +89,15 @@ var data = {}; data[field] = value; db.collection('objects').update({_key:key}, {$set:data}, {upsert:true, w: 1}, function(err, result) { - console.log('SET OBJECT COMPLETE', err, result); callback(err, result); }); } module.getObject = function(key, callback) { - console.log('GET OBJECT', key); db.collection('objects').findOne({_key:key}, function(err, item) { - console.log('RETURNING OBJECT', item); + if(item && item._id) { + delete item._id; + } callback(err, item); }); } @@ -127,17 +129,28 @@ for(var i=0; i Date: Tue, 3 Dec 2013 17:13:59 -0500 Subject: [PATCH 21/83] deleteObjectField, isObjectField in mongo --- src/database/mongo.js | 13 ++++++++++--- src/routes/debug.js | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index d26f33a6ea..571a7b33da 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -116,7 +116,7 @@ var _fields = {}; for(var i=0; i Date: Tue, 3 Dec 2013 17:29:10 -0500 Subject: [PATCH 22/83] completed hashes in mongodb I hope :) --- src/database/mongo.js | 14 ++++++++++---- src/routes/debug.js | 11 ++++++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 571a7b33da..a202a63cea 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -163,21 +163,27 @@ var data = {}; data[field] = ""; db.collection('objects').update({_key:key}, {$unset : data}, function(err, result) { - console.log(err, result); callback(err, result); }); } module.incrObjectField = function(key, field, callback) { - throw new Error('not-implemented'); + module.incrObjectFieldBy(key, field, 1, callback); } module.decrObjectField = function(key, field, callback) { - throw new Error('not-implemented'); + module.incrObjectFieldBy(key, field, -1, callback); } module.incrObjectFieldBy = function(key, field, value, callback) { - throw new Error('not-implemented'); + var data = {}; + data[field] = value; + db.collection('objects').update({_key:key}, {$inc : data}, function(err, result) { + console.log('incrObjectFieldBy', err, result); + module.getObjectField(key, field, function(err, value) { + callback(err, value); + }); + }); } diff --git a/src/routes/debug.js b/src/routes/debug.js index eec77f3402..ac183d0b72 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -141,6 +141,13 @@ var DebugRoute = function(app) { }); } + function incrObjectFieldBy(callback) { + db.incrObjectFieldBy(objectKey, 'age', 3, function(err, data) { + console.log('incrObjectFieldBy return', data); + callback(err, {'incrObjectFieldBy':data}); + }); + } + var tasks = [ setObject, getObject, @@ -153,7 +160,9 @@ var DebugRoute = function(app) { getObjectField, getObjectFields, getObjectValues, - isObjectField + isObjectField, + incrObjectFieldBy, + getObject ]; require('async').series(tasks, function(err, results) { From 3d18c4015a64d636305fc488168186eecc0b3bb1 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Tue, 3 Dec 2013 17:43:12 -0500 Subject: [PATCH 23/83] cleanup --- src/database/mongo.js | 3 +-- src/database/redis.js | 4 ++++ src/plugins.js | 18 +++++++++++++----- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index a202a63cea..d72fbc8e9f 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -109,7 +109,7 @@ } callback(null, data[field]); - }) + }); } module.getObjectFields = function(key, fields, callback) { @@ -179,7 +179,6 @@ var data = {}; data[field] = value; db.collection('objects').update({_key:key}, {$inc : data}, function(err, result) { - console.log('incrObjectFieldBy', err, result); module.getObjectField(key, field, function(err, value) { callback(err, value); }); diff --git a/src/database/redis.js b/src/database/redis.js index 4dc45ae6eb..6a673de6a6 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -136,6 +136,10 @@ redisClient.get(key, callback); } + module.keys = function(key, callback) { + redisClient.keys(keym callback); + } + //hashes module.setObject = function(key, data, callback) { diff --git a/src/plugins.js b/src/plugins.js index 0ba6eae81d..ea0e459980 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -153,13 +153,19 @@ var fs = require('fs'), })); next(); - } else next(); + } else { + next(); + } } ], function(err) { if (!err) { - if (global.env === 'development') winston.info('[plugins] Loaded plugin: ' + pluginData.id); + if (global.env === 'development') { + winston.info('[plugins] Loaded plugin: ' + pluginData.id); + } callback(); - } else callback(new Error('Could not load plugin system')) + } else { + callback(new Error('Could not load plugin system')); + } }); }); }; @@ -212,7 +218,9 @@ var fs = require('fs'), next(null, value); } } else { - if (global.env === 'development') winston.info('[plugins] Expected method \'' + hookObj.method + '\' in plugin \'' + hookObj.id + '\' not found, skipping.'); + if (global.env === 'development') { + winston.info('[plugins] Expected method \'' + hookObj.method + '\' in plugin \'' + hookObj.id + '\' not found, skipping.'); + } next(null, value); } }, function(err, value) { @@ -222,7 +230,7 @@ var fs = require('fs'), } } - callback.apply(plugins, arguments); + callback.apply(Plugins, arguments); }); break; case 'action': From 3dc3769088909e48ad04b0d47c794fc6535c1c8a Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Tue, 3 Dec 2013 17:48:18 -0500 Subject: [PATCH 24/83] fixed redis.keys --- src/database/redis.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/redis.js b/src/database/redis.js index 6a673de6a6..07a2d54945 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -137,7 +137,7 @@ } module.keys = function(key, callback) { - redisClient.keys(keym callback); + redisClient.keys(key, callback); } //hashes From cb6c42ea4422b01f41d002212e282e3895a88032 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Tue, 3 Dec 2013 18:03:50 -0500 Subject: [PATCH 25/83] user stuff --- src/database/mongo.js | 10 ++++++++-- src/routes/debug.js | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index d72fbc8e9f..a158d8632e 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -154,8 +154,14 @@ } module.isObjectField = function(key, field, callback) { - module.getObjectField(key, field, function(err, item) { - callback(err, item !== undefined); + var data = {}; + data[field] = ''; + db.collection('objects').findOne({_key:key}, {fields:data}, function(err, item) { + if(err) { + return callback(err); + } + + callback(err, item && item[field]!== undefined && item[field] !== null); }); } diff --git a/src/routes/debug.js b/src/routes/debug.js index ac183d0b72..e0ca415998 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -85,6 +85,14 @@ var DebugRoute = function(app) { var db = require('./../database'); var objectKey = 'someotherObj'; + function createUser(callback) { + user.create('baris','123456', 'barisusakli@gmail.com', callback); + } + + function getUser(callback) { + user.getUserData(1, callback); + } + function setObject(callback) { db.setObject(objectKey, {name:'baris', 'lastname':'usakli', age:3}, function(err, result) { console.log('setObject return ', result); @@ -149,6 +157,8 @@ var DebugRoute = function(app) { } var tasks = [ + //createUser, + getUser, setObject, getObject, deleteObjectField, From bf3822e8a5f2e59393aaf0306af2a42b7011fff2 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Tue, 3 Dec 2013 18:11:35 -0500 Subject: [PATCH 26/83] incr wil create key if it doesnt exist --- src/database/mongo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index a158d8632e..247b53a480 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -184,7 +184,7 @@ module.incrObjectFieldBy = function(key, field, value, callback) { var data = {}; data[field] = value; - db.collection('objects').update({_key:key}, {$inc : data}, function(err, result) { + db.collection('objects').update({_key:key}, {$inc : data}, {upsert:true}, function(err, result) { module.getObjectField(key, field, function(err, value) { callback(err, value); }); From b5770be71f450392dae5bce4899f2c7529f8a57b Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Tue, 3 Dec 2013 18:19:27 -0500 Subject: [PATCH 27/83] whitespace --- src/database/mongo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 247b53a480..fe81e0fd07 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -184,7 +184,7 @@ module.incrObjectFieldBy = function(key, field, value, callback) { var data = {}; data[field] = value; - db.collection('objects').update({_key:key}, {$inc : data}, {upsert:true}, function(err, result) { + db.collection('objects').update({_key:key}, {$inc : data}, {upsert:true}, function(err, result) { module.getObjectField(key, field, function(err, value) { callback(err, value); }); From 28c75e09a98c888880660babf40de0d16a504c91 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Tue, 3 Dec 2013 22:16:44 -0500 Subject: [PATCH 28/83] can haz sorted sets? --- src/database/mongo.js | 79 ++++++++++++++++++++++++++++++++++++------- src/database/redis.js | 4 +-- src/routes/debug.js | 36 ++++++++++++++++++-- 3 files changed, 101 insertions(+), 18 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index fe81e0fd07..752f291249 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -26,15 +26,13 @@ }); - db.createCollection('objects', function(err, collection) { - }); + db.createCollection('objects', function(err, _collection) { - db.createCollection('sets', function(err, collection) { }); - callback(err); }); + // look up how its done in mongo /*if (nconf.get('mongo:password')) { redisClient.auth(nconf.get('mongo:password')); @@ -68,7 +66,7 @@ throw new Error('not-implemented'); } - module.set = function(key, callback) { + module.set = function(key, value, callback) { throw new Error('not-implemented'); } @@ -124,15 +122,20 @@ return callback(err); } - var data = {}; if(item === null) { + item = {}; + for(var i=0; i Date: Tue, 3 Dec 2013 22:30:36 -0500 Subject: [PATCH 29/83] added sortedSetRemove to mongo --- src/database/mongo.js | 6 ++++-- src/routes/debug.js | 11 ++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 752f291249..ae5a049fad 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -247,12 +247,14 @@ value:value }; - data.setName = key + data.setName = key; module.setObject(key+':'+value, data, callback); } module.sortedSetRemove = function(key, value, callback) { - throw new Error('not-implemented'); + db.collection('objects').remove({setName:key, value:value}, function(err, result) { + callback(err, result); + }); } function getSortedSetRange(key, start, stop, sort, callback) { diff --git a/src/routes/debug.js b/src/routes/debug.js index 85390da963..72391b0aa8 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -164,6 +164,13 @@ var DebugRoute = function(app) { }); } + function sortedSetRemove(callback) { + db.sortedSetRemove('sortedSet2', 12, function(err, data) { + console.log('sortedSetRemove return', data); + callback(err, {'sortedSetRemove': data}); + }); + } + function getSortedSetRange(callback) { db.getSortedSetRevRange('sortedSet2', 0, -1, function(err, data) { console.log('getSortedSetRange return', data); @@ -199,7 +206,9 @@ var DebugRoute = function(app) { //sortedSetAdd, getSortedSetRange, sortedSetAdd, - getSortedSetRange + getSortedSetRange, + sortedSetRemove, + getSortedSetRange, ]; From 113cb85c46aa26ae5f1a752d31703eb4bb935a5e Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 12:10:53 -0500 Subject: [PATCH 30/83] added lists to mongo --- src/database/mongo.js | 36 +++++++++++++++++++++++++++++++++--- src/routes/debug.js | 29 ++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index ae5a049fad..1e8c6ff564 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -318,15 +318,45 @@ // lists module.listPrepend = function(key, value, callback) { - throw new Error('not-implemented'); + module.isObjectField(key, 'array', function(err, exists) { + if(err) { + return callback(err); + } + if(exists) { + db.collection('objects').update({_key:key}, {'$set': {'array.-1': value}}, {upsert:true, w:1 }, function(err, result) { + callback(err, result); + }); + } else { + module.listAppend(key, value, callback); + } + + }) } module.listAppend = function(key, value, callback) { - throw new Error('not-implemented'); + db.collection('objects').update({ _key: key }, { $push: { array: value } }, {upsert:true, w:1}, function(err, result) { + callback(err, result); + }); } module.getListRange = function(key, start, stop, callback) { - throw new Error('not-implemented'); + + if(stop === -1) { + // mongo doesnt allow -1 as the count argument in slice + // pass in a large value to retrieve the whole array + stop = Math.pow(2, 31) - 2; + } + + db.collection('objects').findOne({_key:key}, { array: { $slice: [start, stop - start + 1] }}, function(err, data) { + if(err) { + return callback(err); + } + if(data && data.array) { + callback(err, data.array); + } else { + callback(err, []); + } + }); } diff --git a/src/routes/debug.js b/src/routes/debug.js index 72391b0aa8..d59a922f18 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -183,6 +183,28 @@ var DebugRoute = function(app) { });*/ } + function listAppend(callback) { + db.listAppend('myList5', 5, function(err, data) { + console.log('listAppend return', data); + callback(err, {'listAppend': data}); + }); + } + + function listPrepend(callback) { + db.listPrepend('myList5', 4, function(err, data) { + console.log('listPrepend return', data); + callback(err, {'listPrepend': data}); + }); + } + + function getListRange(callback) { + db.getListRange('myList5', 0, 5, function(err, data) { + console.log('getListRange return', data); + callback(err, {'getListRange': data}); + }); + } + + var objectTasks = [ //createUser, getUser, @@ -211,8 +233,13 @@ var DebugRoute = function(app) { getSortedSetRange, ]; + var listTasks = [ + listAppend, + listPrepend, + getListRange + ]; - require('async').series(sortedSetTasks, function(err, results) { + require('async').series(listTasks, function(err, results) { if(err) { console.log(err); res.send(err.message); From ca01fb9f7d5994989409df1bc9e7060621aa6f83 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 12:25:53 -0500 Subject: [PATCH 31/83] added key methods to mongo --- src/database/mongo.js | 15 +++++++++------ src/routes/debug.js | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 1e8c6ff564..051d7aa930 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -55,19 +55,21 @@ // key module.exists = function(key, callback) { - throw new Error('not-implemented'); + module.isObjectField('global', key, callback); } module.delete = function(key, callback) { - throw new Error('not-implemented'); + module.deleteObjectField('global', key, callback); } module.get = function(key, callback) { - throw new Error('not-implemented'); + module.getObjectField('global', key, callback); } module.set = function(key, value, callback) { - throw new Error('not-implemented'); + var data = {}; + data[key] = value; + module.setObject('global', data, callback); } module.keys = function(key, callback) { @@ -238,6 +240,7 @@ throw new Error('not-implemented'); } + // sorted sets module.sortedSetAdd = function(key, score, value, callback) { @@ -352,9 +355,9 @@ return callback(err); } if(data && data.array) { - callback(err, data.array); + callback(null, data.array); } else { - callback(err, []); + callback(null, []); } }); } diff --git a/src/routes/debug.js b/src/routes/debug.js index d59a922f18..807ea584f5 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -204,6 +204,34 @@ var DebugRoute = function(app) { }); } + function get(callback) { + db.get('testingStr', function(err, data) { + console.log('get return', data); + callback(err, {'get': data}); + }); + } + + function set(callback) { + db.set('testingStr', 'opppa gangastayla', function(err, data) { + console.log('set return', data); + callback(err, {'set': data}); + }); + } + + function deleteKey(callback) { + db.delete('testingStr', function(err, data) { + console.log('delete return', data); + callback(err, {'delete': data}); + }); + } + + function exists(callback) { + db.exists('testingStr', function(err, data) { + console.log('exists return', data); + callback(err, {'exists': data}); + }); + } + var objectTasks = [ //createUser, @@ -239,7 +267,17 @@ var DebugRoute = function(app) { getListRange ]; - require('async').series(listTasks, function(err, results) { + var keyTasks = [ + get, + set, + get, + exists, + deleteKey, + get, + exists + ]; + + require('async').series(keyTasks, function(err, results) { if(err) { console.log(err); res.send(err.message); From 5c6a7d4b94036d757a8816fa31794a431637b84c Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 12:36:22 -0500 Subject: [PATCH 32/83] added regex keys to mongo --- src/database/mongo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 051d7aa930..6b8e02df08 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -73,7 +73,7 @@ } module.keys = function(key, callback) { - throw new Error('not-implemented'); + db.collection.find( { _key: { $regex: key /*, $options: 'i'*/ } } ); } //hashes From 0471a192abff465317593ae9062c96659dcd90c7 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 14:25:14 -0500 Subject: [PATCH 33/83] more mongo stuff --- src/database/mongo.js | 88 ++++++++++++++++++++++++++++++++++--------- src/routes/debug.js | 54 +++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 18 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 6b8e02df08..335726ed85 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -55,25 +55,37 @@ // key module.exists = function(key, callback) { - module.isObjectField('global', key, callback); + db.collection('objects').findOne({_key:key}, function(err, item) { + callback(err, item !== undefined && item !== null); + }); } module.delete = function(key, callback) { - module.deleteObjectField('global', key, callback); + db.collection('objects').remove({_key:key}, function(err, result) { + if(err) { + return callback(err); + } + if(result === 0) { + db.collection('objects').remove({setName:key}, function(err, result) { + callback(err, result); + }); + } else { + callback(null, result); + } + }); } module.get = function(key, callback) { - module.getObjectField('global', key, callback); + module.getObjectField(key, 'value', callback); } module.set = function(key, value, callback) { - var data = {}; - data[key] = value; - module.setObject('global', data, callback); + var data = {value:value}; + module.setObject(key, data, callback); } module.keys = function(key, callback) { - db.collection.find( { _key: { $regex: key /*, $options: 'i'*/ } } ); + db.collection('objects').find( { _key: { $regex: key /*, $options: 'i'*/ } } ); } //hashes @@ -200,15 +212,25 @@ // sets module.setAdd = function(key, value, callback) { - throw new Error('not-implemented'); + + var data = { + value:value + }; + + data.setName = key; + module.setObject(key + ':' + value, data, callback); } module.setRemove = function(key, value, callback) { - throw new Error('not-implemented'); + db.collection('objects').remove({setName:key, value:value}, function(err, result) { + callback(err, result); + }); } module.isSetMember = function(key, value, callback) { - throw new Error('not-implemented'); + db.collection('objects').findOne({setName:key, value:value}, function(err, item) { + callback(err, item !== null && item !== undefined); + }); } module.isMemberOfSets = function(sets, value, callback) { @@ -216,28 +238,51 @@ } module.getSetMembers = function(key, callback) { - console.log('GETTING SET MEMBERS', key); - db.collection('sets').findOne({_key:key}, function(err, data) { + db.collection('objects').find({setName:key}).toArray(function(err, data) { if(err) { return callback(err); } if(!data) { - console.log('GOT SET MEMBERS', []); callback(null, []); } else { - console.log('GOT SET MEMBERS', data); + data = data.map(function(item) { + return item.value; + }); callback(null, data); } }); } module.setCount = function(key, callback) { - throw new Error('not-implemented'); + db.collection('objects').count({setName:key}, function(err, count) { + if(err) { + return callback(err); + } + + if(!count) { + return callback(null, 0); + } + callback(null,count); + }); } module.setRemoveRandom = function(key, callback) { - throw new Error('not-implemented'); + db.collection('objects').find({setName:key}).toArray(function(err, data) { + if(err) { + return callback(err); + } + + if(!data) { + callback(null, 0); + } else { + var randomIndex = Math.floor(Math.random() * data.length); + var item = data[randomIndex]; + module.setRemove(item.setName, item.value, function(err, result) { + callback(err, item.value); + }); + } + }); } @@ -316,7 +361,16 @@ } module.sortedSetCount = function(key, min, max, callback) { - throw new Error('not-implemented'); + db.collection('objects').count({setName:key, score: {$gt:min, $lt:max}}, function(err, count) { + if(err) { + return callback(err); + } + + if(!count) { + return callback(null, 0); + } + callback(null,count); + }); } // lists diff --git a/src/routes/debug.js b/src/routes/debug.js index 807ea584f5..81bc24c9d4 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -233,6 +233,43 @@ var DebugRoute = function(app) { } + function setAdd(callback) { + db.setAdd('myTestSet', 5, function(err, data) { + console.log('setAdd return', data); + callback(err, {'setAdd': data}); + }); + } + + function setRemove(callback) { + db.setRemove('myTestSet', 11, function(err, data) { + console.log('setRemove return', data); + callback(err, {'setRemove': data}); + }); + } + + function getSetMembers(callback) { + db.getSetMembers('myTestSet', function(err, data) { + console.log('getSetMembers return', data); + callback(err, {'getSetMembers': data}); + }); + } + + function isSetMember(callback) { + db.isSetMember('myTestSet', 5, function(err, data) { + console.log('isSetMember return', data); + callback(err, {'isSetMember': data}); + }); + } + + function setRemoveRandom(callback) { + db.setRemoveRandom('myTestSet', function(err, data) { + console.log('setRemoveRandom return', data); + callback(err, {'setRemoveRandom': data}); + }); + } + + + var objectTasks = [ //createUser, getUser, @@ -273,11 +310,26 @@ var DebugRoute = function(app) { get, exists, deleteKey, + deleteKey, get, exists ]; - require('async').series(keyTasks, function(err, results) { + var setTasks = [ + getSetMembers, + // setRemove, + setAdd, + setAdd, + getSetMembers, + isSetMember, + // setRemove, + isSetMember, + getSetMembers, + setRemoveRandom, + getSetMembers + ]; + + require('async').series(setTasks, function(err, results) { if(err) { console.log(err); res.send(err.message); From cfd3a7d12686879466db91731f44e8f254f65804 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 14:51:50 -0500 Subject: [PATCH 34/83] added flushdb command to both dbs --- src/database/mongo.js | 112 +++++++++++++++++++++++++++++++++--------- src/database/redis.js | 11 +++++ 2 files changed, 99 insertions(+), 24 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 335726ed85..0bdfd5a442 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -13,7 +13,6 @@ module.init = function(callback) { mongoClient.connect('mongodb://'+ mongoHost + ':' + nconf.get('mongo:port') + '/' + nconf.get('mongo:database'), function(err, _db) { db = _db; - console.log('WE ARE CONNECTED'); if(err) { winston.error("NodeBB could not connect to your Mongo database. Mongo returned the following error: " + err.message); @@ -44,6 +43,18 @@ // // Exported functions // + + module.flushdb = function(callback) { + db.dropDatabase(function(err, result) { + if(err){ + winston.error(error); + return callback(err); + } + callback(null); + }); + } + + module.getFileName = function(callback) { throw new Error('not-implemented'); } @@ -63,14 +74,23 @@ module.delete = function(key, callback) { db.collection('objects').remove({_key:key}, function(err, result) { if(err) { - return callback(err); + if(callback) { + return callback(err); + } else { + return winston.error(err.message); + } } + if(result === 0) { db.collection('objects').remove({setName:key}, function(err, result) { - callback(err, result); + if(callback) { + callback(err, result); + } }); } else { - callback(null, result); + if(callback) { + callback(null, result); + } } }); } @@ -85,15 +105,33 @@ } module.keys = function(key, callback) { - db.collection('objects').find( { _key: { $regex: key /*, $options: 'i'*/ } } ); + db.collection('objects').find( { _key: { $regex: key /*, $options: 'i'*/ } }, function(err, result) { + callback(err, result); + }); } //hashes + function removeHiddenFields(item) { + if(item) { + if(item._id) { + delete item._id; + } + if(item._key) { + delete item._key; + } + if(item.setName) { + delete item.setName; + } + } + return item; + } module.setObject = function(key, data, callback) { data['_key'] = key; db.collection('objects').update({_key:key}, {$set:data}, {upsert:true, w: 1}, function(err, result) { - callback(err, result); + if(callback) { + callback(err, result); + } }); } @@ -101,15 +139,16 @@ var data = {}; data[field] = value; db.collection('objects').update({_key:key}, {$set:data}, {upsert:true, w: 1}, function(err, result) { - callback(err, result); + if(callback) { + callback(err, result); + } }); } module.getObject = function(key, callback) { db.collection('objects').findOne({_key:key}, function(err, item) { - if(item && item._id) { - delete item._id; - } + removeHiddenFields(item); + callback(err, item); }); } @@ -131,7 +170,8 @@ _fields[fields[i]] = 1; } - db.collection('objects').findOne({_key:key}, {fields:_fields}, function(err, item) { + db.collection('objects').findOne({_key:key}, _fields, function(err, item) { + if(err) { return callback(err); } @@ -149,9 +189,8 @@ } } - if(item && item._id) { - delete item._id; - } + removeHiddenFields(item); + callback(err, item); }); } @@ -186,7 +225,9 @@ var data = {}; data[field] = ""; db.collection('objects').update({_key:key}, {$unset : data}, function(err, result) { - callback(err, result); + if(callback) { + callback(err, result); + } }); } @@ -203,7 +244,9 @@ data[field] = value; db.collection('objects').update({_key:key}, {$inc : data}, {upsert:true}, function(err, result) { module.getObjectField(key, field, function(err, value) { - callback(err, value); + if(callback) { + callback(err, value); + } }); }); } @@ -223,7 +266,9 @@ module.setRemove = function(key, value, callback) { db.collection('objects').remove({setName:key, value:value}, function(err, result) { - callback(err, result); + if(callback) { + callback(err, result); + } }); } @@ -270,16 +315,24 @@ module.setRemoveRandom = function(key, callback) { db.collection('objects').find({setName:key}).toArray(function(err, data) { if(err) { - return callback(err); + if(callback) { + return callback(err); + } else { + return winston.error(err.message); + } } if(!data) { - callback(null, 0); + if(callback) { + callback(null, 0); + } } else { var randomIndex = Math.floor(Math.random() * data.length); var item = data[randomIndex]; module.setRemove(item.setName, item.value, function(err, result) { - callback(err, item.value); + if(callback) { + callback(err, item.value); + } }); } }); @@ -301,7 +354,9 @@ module.sortedSetRemove = function(key, value, callback) { db.collection('objects').remove({setName:key, value:value}, function(err, result) { - callback(err, result); + if(callback) { + callback(err, result); + } }); } @@ -377,11 +432,18 @@ module.listPrepend = function(key, value, callback) { module.isObjectField(key, 'array', function(err, exists) { if(err) { - return callback(err); + if(callback) { + return callback(err); + } else { + return winston.error(err.message); + } } + if(exists) { db.collection('objects').update({_key:key}, {'$set': {'array.-1': value}}, {upsert:true, w:1 }, function(err, result) { - callback(err, result); + if(callback) { + callback(err, result); + } }); } else { module.listAppend(key, value, callback); @@ -392,7 +454,9 @@ module.listAppend = function(key, value, callback) { db.collection('objects').update({ _key: key }, { $push: { array: value } }, {upsert:true, w:1}, function(err, result) { - callback(err, result); + if(callback) { + callback(err, result); + } }); } diff --git a/src/database/redis.js b/src/database/redis.js index 57bc1e02b2..95b1909936 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -64,6 +64,17 @@ // // Exported functions // + + module.flushdb = function(callback) { + redisClient.send_command('flushdb', [], function(err) { + if(err){ + winston.error(error); + return callback(err); + } + callback(null); + }); + } + module.getFileName = function(callback) { var multi = redisClient.multi(); From 4b5988c269263d6de71d7486bdcf0dd5b1b4a9c1 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 15:11:17 -0500 Subject: [PATCH 35/83] isSetMember returns true or false --- src/database/mongo.js | 31 +++++++++++++++++++++---------- src/database/redis.js | 8 +++++++- src/routes/debug.js | 14 ++++++++------ src/threadTools.js | 2 +- src/user.js | 8 ++++---- 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 0bdfd5a442..3cc0b5c907 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -4,6 +4,7 @@ 'use strict'; var mongoClient = require('mongodb').MongoClient, winston = require('winston'), + async = require('async'), nconf = require('nconf'), express = require('express'), mongoStore = require('connect-mongo')(express), @@ -279,7 +280,17 @@ } module.isMemberOfSets = function(sets, value, callback) { - throw new Error('not-implemented'); + + + function iterator(set, next) { + module.isSetMember(set, value, next); + } + + async.map(sets, iterator, function(err, result) { + console.log(err, result); + callback(err, result); + }); + } module.getSetMembers = function(key, callback) { @@ -469,15 +480,15 @@ } db.collection('objects').findOne({_key:key}, { array: { $slice: [start, stop - start + 1] }}, function(err, data) { - if(err) { - return callback(err); - } - if(data && data.array) { - callback(null, data.array); - } else { - callback(null, []); - } - }); + if(err) { + return callback(err); + } + if(data && data.array) { + callback(null, data.array); + } else { + callback(null, []); + } + }); } diff --git a/src/database/redis.js b/src/database/redis.js index 95b1909936..c640a5530b 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -234,7 +234,13 @@ } module.isSetMember = function(key, value, callback) { - redisClient.sismember(key, value, callback); + redisClient.sismember(key, value, function(err, result) { + if(err) { + return callback(err); + } + + callback(null, result === 1); + }); } module.isMemberOfSets = function(sets, value, callback) { diff --git a/src/routes/debug.js b/src/routes/debug.js index 81bc24c9d4..76f28036a3 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -261,6 +261,13 @@ var DebugRoute = function(app) { }); } + function isMemberOfSets(callback) { + db.isMemberOfSets(['doesntexist', 'myTestSet', 'nonexistingSet'], 5, function(err, data) { + console.log('isMemberOfSets return', data); + callback(err, {'isMemberOfSets': data}); + }); + } + function setRemoveRandom(callback) { db.setRemoveRandom('myTestSet', function(err, data) { console.log('setRemoveRandom return', data); @@ -317,16 +324,11 @@ var DebugRoute = function(app) { var setTasks = [ getSetMembers, - // setRemove, - setAdd, setAdd, getSetMembers, isSetMember, - // setRemove, - isSetMember, getSetMembers, - setRemoveRandom, - getSetMembers + isMemberOfSets ]; require('async').series(setTasks, function(err, results) { diff --git a/src/threadTools.js b/src/threadTools.js index 331bc71298..07dc482884 100644 --- a/src/threadTools.js +++ b/src/threadTools.js @@ -22,7 +22,7 @@ var db = require('./database'), callback(false); } - callback( !! ismember || false); + callback(ismember); }); } diff --git a/src/user.js b/src/user.js index 686b8e39c3..164efd6db1 100644 --- a/src/user.js +++ b/src/user.js @@ -581,9 +581,9 @@ var bcrypt = require('bcrypt'), }; User.isFollowing = function(uid, theirid, callback) { - db.isSetMember('following:' + uid, theirid, function(err, data) { + db.isSetMember('following:' + uid, theirid, function(err, isMember) { if (!err) { - callback(data === 1); + callback(isMember); } else { console.log(err); } @@ -695,14 +695,14 @@ var bcrypt = require('bcrypt'), if(err) { return calback(err); } - callback(err, !! exists); + callback(err, exists); }); }; User.isAdministrator = function(uid, callback) { groups.getGidFromName('Administrators', function(err, gid) { groups.isMember(uid, gid, function(err, isAdmin) { - callback(err, !! isAdmin); + callback(err, isAdmin); }); }); }; From b547d3577bd015f41adf5a05ada9ffc52163dba5 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 15:13:43 -0500 Subject: [PATCH 36/83] removed a multi from categories.js --- src/categories.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/categories.js b/src/categories.js index 31295405e0..538bc0d5c3 100644 --- a/src/categories.js +++ b/src/categories.js @@ -193,13 +193,14 @@ var db = require('./database.js'), }; Categories.hasReadCategories = function(cids, uid, callback) { - var batch = RDB.multi(); + + var sets = []; for (var i = 0, ii = cids.length; i < ii; i++) { - batch.sismember('cid:' + cids[i] + ':read_by_uid', uid); + sets.push('cid:' + cids[i] + ':read_by_uid'); } - batch.exec(function(err, hasRead) { + db.isMemberOfSets(sets, uid, function(err, hasRead) { callback(hasRead); }); }; From 53a7eab3e8fd6b93b3abac99eafdfb119578e367 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 16:31:05 -0500 Subject: [PATCH 37/83] removed couple RDB.multis --- src/database/mongo.js | 34 ++++++++++++++++++++++++++++++++++ src/database/redis.js | 9 +++++++++ src/notifications.js | 32 +++++++++++++------------------- src/routes/debug.js | 19 +++++++++++++++---- src/user.js | 15 ++++++++------- tests/categories.js | 2 ++ 6 files changed, 81 insertions(+), 30 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 3cc0b5c907..9ad149c3eb 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -439,6 +439,20 @@ }); } + module.sortedSetRank = function(key, value, callback) { + module.getSortedSetRange(key, 0, -1, function(err, result) { + if(err) { + return callback(err); + } + var rank = result.indexOf(value); + if(rank === -1) { + return callback(null, null); + } + + callback(null, rank); + }); + } + // lists module.listPrepend = function(key, value, callback) { module.isObjectField(key, 'array', function(err, exists) { @@ -471,6 +485,26 @@ }); } + module.listRemoveLast = function(key, callback) { + module.getListRange(key, -1, 0, function(err, value) { + if(err) { + return callback(err); + } + + db.collection('objects').update({_key: key }, { $pop: { array: 1 } }, function(err, result) { + if(err) { + return callback(err); + } + + if(value && value.length) { + callback(err, value[0]); + } else { + callback(err, null); + } + }); + }); + } + module.getListRange = function(key, start, stop, callback) { if(stop === -1) { diff --git a/src/database/redis.js b/src/database/redis.js index c640a5530b..a5ca2f78f4 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -291,6 +291,10 @@ redisClient.zcount(key, min, max, callback); } + module.sortedSetRank = function(key, value, callback) { + redisClient.zrank(key, value, callback); + } + // lists module.listPrepend = function(key, value, callback) { redisClient.lpush(key, value, callback); @@ -300,11 +304,16 @@ redisClient.rpush(key, value, callback); } + module.listRemoveLast = function(key, callback) { + redisClient.rpop(key, callback); + } + module.getListRange = function(key, start, stop, callback) { redisClient.lrange(key, start, stop, callback); } + }(exports)); diff --git a/src/notifications.js b/src/notifications.js index a6f8fc64a9..ec658b57d9 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -18,28 +18,22 @@ var async = require('async'), }; Notifications.get = function(nid, uid, callback) { - RDB.multi() - .hmget('notifications:' + nid, 'text', 'score', 'path', 'datetime', 'uniqueId') - .zrank('uid:' + uid + ':notifications:read', nid) - .exists('notifications:' + nid) - .exec(function(err, results) { - var notification = results[0], - readIdx = results[1]; - - if (!results[2]) { - return callback(null); - } - callback({ - nid: nid, - text: notification[0], - score: notification[1], - path: notification[2], - datetime: notification[3], - uniqueId: notification[4], - read: readIdx !== null ? true : false + db.exists('nofitications:' + nid, function(err, exists) { + + if(!exists) { + return callback(null); + } + + db.sortedSetRank('uid:' + uid + ':notifications:read', nid, function(err, rank) { + + db.getObjectFields('notifications:' + nid, ['nid', 'text', 'score', 'path', 'datetime', 'uniqueId'], function(err, notification) { + + notification.read = rank !== null ? true:false; + callback(notification); }); }); + }); }; Notifications.create = function(text, path, uniqueId, callback) { diff --git a/src/routes/debug.js b/src/routes/debug.js index 76f28036a3..d475ee8774 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -197,8 +197,17 @@ var DebugRoute = function(app) { }); } + + function listRemoveLast(callback) { + db.listRemoveLast('myList5', function(err, data) { + console.log('listRemoveLast return', data); + callback(err, {'listRemoveLast': data}); + }); + } + + function getListRange(callback) { - db.getListRange('myList5', 0, 5, function(err, data) { + db.getListRange('myList5', 0, -1, function(err, data) { console.log('getListRange return', data); callback(err, {'getListRange': data}); }); @@ -306,8 +315,10 @@ var DebugRoute = function(app) { ]; var listTasks = [ - listAppend, - listPrepend, + //listAppend, + //listPrepend, + getListRange, + listRemoveLast, getListRange ]; @@ -331,7 +342,7 @@ var DebugRoute = function(app) { isMemberOfSets ]; - require('async').series(setTasks, function(err, results) { + require('async').series(listTasks, function(err, results) { if(err) { console.log(err); res.send(err.message); diff --git a/src/user.js b/src/user.js index 164efd6db1..7770f8862d 100644 --- a/src/user.js +++ b/src/user.js @@ -1005,14 +1005,13 @@ var bcrypt = require('bcrypt'), before = new Date(parseInt(before, 10)); } - // TODO : figure out how to do this with dbal - RDB.multi() - .zrevrangebyscore('uid:' + uid + ':notifications:read', before ? before.getTime(): now.getTime(), -Infinity, 'LIMIT', 0, limit) - .zrevrangebyscore('uid:' + uid + ':notifications:unread', before ? before.getTime(): now.getTime(), -Infinity, 'LIMIT', 0, limit) - .exec(function(err, results) { - // Merge the read and unread notifications - var nids = results[0].concat(results[1]); + var args1 = ['uid:' + uid + ':notifications:read', before ? before.getTime(): now.getTime(), -Infinity, 'LIMIT', 0, limit]; + var args2 = ['uid:' + uid + ':notifications:unread', before ? before.getTime(): now.getTime(), -Infinity, 'LIMIT', 0, limit]; + db.getSortedSetRevRangeByScore(args1, function(err, results1) { + db.getSortedSetRevRangeByScore(args2, function(err, results2) { + + var nids = results1.concat(results2); async.map(nids, function(nid, next) { notifications.get(nid, uid, function(notif_data) { next(null, notif_data); @@ -1032,6 +1031,8 @@ var bcrypt = require('bcrypt'), callback(err, notifs); }); }); + }); + }, getUnreadCount: function(uid, callback) { db.sortedSetCount('uid:' + uid + ':notifications:unread', 0, 10, callback); diff --git a/tests/categories.js b/tests/categories.js index 11456b54a4..36a4d63925 100644 --- a/tests/categories.js +++ b/tests/categories.js @@ -62,9 +62,11 @@ describe('Categories', function() { }); after(function() { + // TODO : replace with dbal RDB.multi() .del('category:'+categoryObj.cid) .rpop('categories:cid') .exec(); + }); }); \ No newline at end of file From 2e2938616d183e808804bd2d08c84ab103af03e2 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 16:58:06 -0500 Subject: [PATCH 38/83] more mongo and redis stuff --- src/database/mongo.js | 7 +++++++ src/database/redis.js | 12 ++++++++++++ src/notifications.js | 21 ++++++++++++--------- src/posts.js | 17 ++++++++--------- src/routes/debug.js | 12 ++++++++++-- src/threadTools.js | 20 +++++++++----------- src/webserver.js | 1 - 7 files changed, 58 insertions(+), 32 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 9ad149c3eb..f5e9c7706e 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -154,6 +154,13 @@ }); } + module.getObjects = function(keys, callback) { + db.collection('objects').find({_key:{$in:keys}}, {_id:0, _key:0}).toArray(function(err, data) { + + callback(err, data); + }); + } + module.getObjectField = function(key, field, callback) { module.getObjectFields(key, [field], function(err, data) { if(err) { diff --git a/src/database/redis.js b/src/database/redis.js index a5ca2f78f4..6ce97d83cb 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -170,6 +170,18 @@ redisClient.hgetall(key, callback); } + module.getObjects = function(keys, callback) { + var multi = redisClient.multi(); + + for(var x=0; x Date: Wed, 4 Dec 2013 17:08:49 -0500 Subject: [PATCH 39/83] commented out reds in install --- src/install.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/install.js b/src/install.js index 3f12f4b9fb..ee1d3f08a7 100644 --- a/src/install.js +++ b/src/install.js @@ -375,10 +375,10 @@ var async = require('async'), file: path.join(__dirname, '..', 'config.json') }); - var RDB = require('./redis'); + /*var RDB = require('./redis'); reds.createClient = function () { return reds.client || (reds.client = RDB); - }; + };*/ callback(err); }); From b215dbde1953fbd726e83d34a8a0ed76943dcd3b Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 17:57:51 -0500 Subject: [PATCH 40/83] updated install --- src/database/redis.js | 2 +- src/install.js | 125 ++++++++++++++++++++++++++++-------------- src/meta.js | 4 +- src/plugins.js | 8 ++- 4 files changed, 93 insertions(+), 46 deletions(-) diff --git a/src/database/redis.js b/src/database/redis.js index 6ce97d83cb..cfb34dad10 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -163,7 +163,7 @@ } module.setObjectField = function(key, field, value, callback) { - redisClient.hset(key, field, value, callback) + redisClient.hset(key, field, value, callback); } module.getObject = function(key, callback) { diff --git a/src/install.js b/src/install.js index ee1d3f08a7..22d00cfdd3 100644 --- a/src/install.js +++ b/src/install.js @@ -19,8 +19,8 @@ var async = require('async'), name: 'port', description: 'Port number of your NodeBB', 'default': nconf.get('port') || 4567, - pattern: /[0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]/, - message: 'Please enter a value betweeen 1 and 65535' + pattern: /[0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]/, + message: 'Please enter a value betweeen 1 and 65535' }, { name: 'use_port', description: 'Use a port number to access NodeBB?', @@ -32,6 +32,15 @@ var async = require('async'), description: 'Please enter a NodeBB secret', 'default': nconf.get('secret') || utils.generateUUID() }, { + name: 'bind_address', + description: 'IP or Hostname to bind to', + 'default': nconf.get('bind_address') || '0.0.0.0' + }, { + name: 'database', + description: 'Which database to use', + 'default': nconf.get('database') || 'redis' + }], + redisQuestions : [{ name: 'redis:host', description: 'Host IP or address of your Redis instance', 'default': nconf.get('redis:host') || '127.0.0.1' @@ -46,11 +55,24 @@ var async = require('async'), name: "redis:database", description: "Which database to use (0..n)", 'default': nconf.get('redis:database') || 0 + }], + mongoQuestions : [{ + name: 'mongo:host', + description: 'Host IP or address of your MongoDB instance', + 'default': nconf.get('mongo:host') || '127.0.0.1' }, { - name: 'bind_address', - description: 'IP or Hostname to bind to', - 'default': nconf.get('bind_address') || '0.0.0.0' + name: 'mongo:port', + description: 'Host port of your MongoDB instance', + 'default': nconf.get('mongo:port') || 27017 + }, { + name: 'mongo:password', + description: 'Password of your MongoDB database' + }, { + name: "mongo:database", + description: "Which database to use (0..n)", + 'default': nconf.get('mongo:database') || 0 }], + setup: function (callback) { async.series([ function(next) { @@ -74,7 +96,9 @@ var async = require('async'), if (!setupVal['admin:email']) winston.error(' admin:email'); process.exit(); } - } else next(); + } else { + next(); + } }, function (next) { var success = function(err, config) { @@ -82,36 +106,54 @@ var async = require('async'), return next(new Error('aborted')); } - // Translate redis properties into redis object - config.redis = { - host: config['redis:host'], - port: config['redis:port'], - password: config['redis:password'], - database: config['redis:database'] - }; - delete config['redis:host']; - delete config['redis:port']; - delete config['redis:password']; - delete config['redis:database']; - - // Add hardcoded values - config.bcrypt_rounds = 12; - config.upload_path = '/public/uploads'; - config.use_port = (config.use_port.slice(0, 1) === 'y') ? true : false; - - var urlObject = url.parse(config.base_url), - relative_path = (urlObject.pathname && urlObject.pathname.length > 1) ? urlObject.pathname : '', - host = urlObject.host, - protocol = urlObject.protocol, - server_conf = config, - client_conf = { - relative_path: relative_path - }; - - server_conf.base_url = protocol + '//' + host; - server_conf.relative_path = relative_path; - - install.save(server_conf, client_conf, next); + function dbQuestionsSuccess(err, databaseConfig) { + if (!config) { + return next(new Error('aborted')); + } + + // Translate redis properties into redis object + if(config.database === 'redis') { + config.redis = { + host: databaseConfig['redis:host'], + port: databaseConfig['redis:port'], + password: databaseConfig['redis:password'], + database: databaseConfig['redis:database'] + }; + } else if (config.database === 'mongo') { + config.mongo = { + host: databaseConfig['mongo:host'], + port: databaseConfig['mongo:port'], + password: databaseConfig['mongo:password'], + database: databaseConfig['mongo:database'] + }; + } + + // Add hardcoded values + config.bcrypt_rounds = 12; + config.upload_path = '/public/uploads'; + config.use_port = (config.use_port.slice(0, 1) === 'y') ? true : false; + + var urlObject = url.parse(config.base_url), + relative_path = (urlObject.pathname && urlObject.pathname.length > 1) ? urlObject.pathname : '', + host = urlObject.host, + protocol = urlObject.protocol, + server_conf = config, + client_conf = { + relative_path: relative_path + }; + + server_conf.base_url = protocol + '//' + host; + server_conf.relative_path = relative_path; + + install.save(server_conf, client_conf, function(err) { + require('./database').init(next); + }); + } + + if(config.database === 'redis') + prompt.get(install.redisQuestions, dbQuestionsSuccess); + else if(config.database === 'mongo') + prompt.get(install.mongoQuestions, dbQuestionsSuccess); }; // prompt prepends "prompt: " to questions, let's clear that. @@ -119,8 +161,9 @@ var async = require('async'), prompt.message = ''; prompt.delimiter = ''; - if (!install.values) prompt.get(install.questions, success); - else { + if (!install.values) { + prompt.get(install.questions, success); + } else { // Use provided values, fall back to defaults var config = {}, question, x, numQ; @@ -176,6 +219,7 @@ var async = require('async'), async.each(defaults, function (configObj, next) { meta.configs.setOnEmpty(configObj.field, configObj.value, next); }, function (err) { + console.log('calling meta.configs.init'); meta.configs.init(next); }); }, @@ -343,8 +387,9 @@ var async = require('async'), // Add the password questions questions = questions.concat(passwordQuestions); - if (!install.values) prompt.get(questions, success); - else { + if (!install.values) { + prompt.get(questions, success); + } else { var results = { username: install.values['admin:username'], email: install.values['admin:email'], diff --git a/src/meta.js b/src/meta.js index 8930c944ba..d75ce18d43 100644 --- a/src/meta.js +++ b/src/meta.js @@ -44,7 +44,7 @@ var fs = require('fs'), db.getObjectFields('config', fields, callback); }, set: function (field, value, callback) { - db.setObjectField(field, value, function(err, res) { + db.setObjectField('config', field, value, function(err, res) { if (callback) { if(!err && Meta.config) Meta.config[field] = value; @@ -53,7 +53,7 @@ var fs = require('fs'), }); }, setOnEmpty: function (field, value, callback) { - this.get(field, function (err, curValue) { + Meta.configs.get(field, function (err, curValue) { if (!curValue) { Meta.configs.set(field, value, callback); } else { diff --git a/src/plugins.js b/src/plugins.js index ea0e459980..ca7114da82 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -275,9 +275,9 @@ var fs = require('fs'), } // Reload meta data - plugins.reload(function() { + Plugins.reload(function() { // (De)activation Hooks - plugins.fireHook('action:plugin.' + (active ? 'de' : '') + 'activate', id); + Plugins.fireHook('action:plugin.' + (active ? 'de' : '') + 'activate', id); if (callback) { callback({ @@ -326,7 +326,9 @@ var fs = require('fs'), } Plugins.isActive(config.id, function(err, active) { - if (err) next(new Error('no-active-state')); + if (err) { + next(new Error('no-active-state')); + } delete config.library; delete config.hooks; From e066fbf36ac99cc015b5de00be4031b4b3ba5f09 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 18:26:26 -0500 Subject: [PATCH 41/83] sets are storing numbers in mongo as opposed to redis which stores strings, causes tons of problems --- src/database/mongo.js | 3 --- src/groups.js | 11 ++++++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index f5e9c7706e..0513248b85 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -287,14 +287,11 @@ } module.isMemberOfSets = function(sets, value, callback) { - - function iterator(set, next) { module.isSetMember(set, value, next); } async.map(sets, iterator, function(err, result) { - console.log(err, result); callback(err, result); }); diff --git a/src/groups.js b/src/groups.js index 98faf0bcbb..972cb9545f 100644 --- a/src/groups.js +++ b/src/groups.js @@ -146,7 +146,16 @@ if (!exists) { db.incrObjectField('global', 'nextGid', function (err, gid) { db.setObjectField('group:gid', name, gid, function(err) { - db.setObject('gid:' + gid, {}, function(err) { + + var groupData = { + gid: gid, + name: name, + description: description, + deleted: '0', + hidden: '0' + }; + + db.setObject('gid:' + gid, groupData, function(err) { Groups.get(gid, {}, callback); From f0caac242c3ce08a1608c1fc5b891be600cd63f5 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 18:34:58 -0500 Subject: [PATCH 42/83] fixed incr in topic post --- src/topics.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topics.js b/src/topics.js index 1f33d5be51..4345545685 100644 --- a/src/topics.js +++ b/src/topics.js @@ -94,7 +94,7 @@ var async = require('async'), // in future it may be possible to add topics to several categories, so leaving the door open here. db.sortedSetAdd('categories:' + cid + ':tid', timestamp, tid); - db.incrObjectField('category:' + cid, 'topic_count', 1); + db.incrObjectField('category:' + cid, 'topic_count'); db.incrObjectField('global', 'topicCount'); feed.updateCategory(cid); From 639247a8b0b9919049c91f8865d0ee75599fa77b Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 18:58:20 -0500 Subject: [PATCH 43/83] fixed notifications --- src/notifications.js | 3 +-- src/user.js | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/notifications.js b/src/notifications.js index a75de443c4..1c2a139252 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -19,8 +19,7 @@ var async = require('async'), Notifications.get = function(nid, uid, callback) { - db.exists('nofitications:' + nid, function(err, exists) { - + db.exists('notifications:' + nid, function(err, exists) { if(!exists) { return callback(null); } diff --git a/src/user.js b/src/user.js index 7770f8862d..74c370d149 100644 --- a/src/user.js +++ b/src/user.js @@ -1039,8 +1039,12 @@ var bcrypt = require('bcrypt'), }, getUnreadByUniqueId: function(uid, uniqueId, callback) { db.getSortedSetRange('uid:' + uid + ':notifications:unread', 0, -1, function(err, nids) { + async.filter(nids, function(nid, next) { notifications.get(nid, uid, function(notifObj) { + if(!notifObj) { + next(false); + } if (notifObj.uniqueId === uniqueId) { next(true); } else { From 72a3ab1d6c12ef065e34ec9e6e780dee712beed6 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 19:06:36 -0500 Subject: [PATCH 44/83] derp --- src/routes/debug.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/routes/debug.js b/src/routes/debug.js index a72ba5bf9c..99f2f584c9 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -359,8 +359,6 @@ var DebugRoute = function(app) { } }); }); - - }); }; From 7c4347736c20cd8818187581ebf14b3d47b29e97 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 21:35:38 -0500 Subject: [PATCH 45/83] added index on _key --- src/database/mongo.js | 8 ++++++-- src/routes/debug.js | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 0513248b85..54802b0eda 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -26,8 +26,12 @@ }); - db.createCollection('objects', function(err, _collection) { - + db.createCollection('objects', function(err, collection) { + collection.ensureIndex({_key :1}, {background:true}, function(err, name){ + if(err) { + winston.error("Error creating index " + err.message); + } + }); }); callback(err); diff --git a/src/routes/debug.js b/src/routes/debug.js index 99f2f584c9..812f49d1da 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -350,7 +350,7 @@ var DebugRoute = function(app) { isMemberOfSets ]; - require('async').series(objectTasks, function(err, results) { + require('async').series(setTasks, function(err, results) { if(err) { console.log(err); res.send(err.message); From 9b557cafd86b2a8c41ffadffec9dac9a823d9e7c Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 22:19:11 -0500 Subject: [PATCH 46/83] better sets in mongo, should change sorted sets too using sort --- src/database/mongo.js | 41 ++++++++++++++++++----------------------- src/routes/debug.js | 22 ++++++++++++++++++---- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 54802b0eda..6e2db0c4f0 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -267,17 +267,15 @@ // sets module.setAdd = function(key, value, callback) { - - var data = { - value:value - }; - - data.setName = key; - module.setObject(key + ':' + value, data, callback); + db.collection('objects').update({_key:key}, {$addToSet: { members: value }}, {upsert:true, w: 1}, function(err, result) { + if(callback) { + callback(err, result); + } + }); } module.setRemove = function(key, value, callback) { - db.collection('objects').remove({setName:key, value:value}, function(err, result) { + db.collection('objects').update({_key:key, members: value}, {$pull : {members: value}}, function(err, result) { if(callback) { callback(err, result); } @@ -285,7 +283,7 @@ } module.isSetMember = function(key, value, callback) { - db.collection('objects').findOne({setName:key, value:value}, function(err, item) { + db.collection('objects').findOne({_key:key, members: value}, function(err, item) { callback(err, item !== null && item !== undefined); }); } @@ -302,7 +300,7 @@ } module.getSetMembers = function(key, callback) { - db.collection('objects').find({setName:key}).toArray(function(err, data) { + db.collection('objects').findOne({_key:key}, {members:1}, function(err, data) { if(err) { return callback(err); } @@ -310,29 +308,26 @@ if(!data) { callback(null, []); } else { - data = data.map(function(item) { - return item.value; - }); - callback(null, data); + callback(null, data.members); } }); } module.setCount = function(key, callback) { - db.collection('objects').count({setName:key}, function(err, count) { + db.collection('objects').findOne({_key:key}, function(err, data) { if(err) { return callback(err); } - - if(!count) { + if(!data) { return callback(null, 0); } - callback(null,count); + + callback(null, data.members.length); }); } module.setRemoveRandom = function(key, callback) { - db.collection('objects').find({setName:key}).toArray(function(err, data) { + db.collection('objects').findOne({_key:key}, function(err, data) { if(err) { if(callback) { return callback(err); @@ -346,11 +341,11 @@ callback(null, 0); } } else { - var randomIndex = Math.floor(Math.random() * data.length); - var item = data[randomIndex]; - module.setRemove(item.setName, item.value, function(err, result) { + var randomIndex = Math.floor(Math.random() * data.members.length); + var value = data.members[randomIndex]; + module.setRemove(data._key, value, function(err, result) { if(callback) { - callback(err, item.value); + callback(err, value); } }); } diff --git a/src/routes/debug.js b/src/routes/debug.js index 812f49d1da..b413e2b5cd 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -250,14 +250,14 @@ var DebugRoute = function(app) { function setAdd(callback) { - db.setAdd('myTestSet', 5, function(err, data) { + db.setAdd('myTestSet', 15, function(err, data) { console.log('setAdd return', data); callback(err, {'setAdd': data}); }); } function setRemove(callback) { - db.setRemove('myTestSet', 11, function(err, data) { + db.setRemove('myTestSet', 15, function(err, data) { console.log('setRemove return', data); callback(err, {'setRemove': data}); }); @@ -271,14 +271,14 @@ var DebugRoute = function(app) { } function isSetMember(callback) { - db.isSetMember('myTestSet', 5, function(err, data) { + db.isSetMember('myTestSet', 15, function(err, data) { console.log('isSetMember return', data); callback(err, {'isSetMember': data}); }); } function isMemberOfSets(callback) { - db.isMemberOfSets(['doesntexist', 'myTestSet', 'nonexistingSet'], 5, function(err, data) { + db.isMemberOfSets(['doesntexist', 'myTestSet', 'nonexistingSet'], 15, function(err, data) { console.log('isMemberOfSets return', data); callback(err, {'isMemberOfSets': data}); }); @@ -291,6 +291,13 @@ var DebugRoute = function(app) { }); } + function setCount(callback) { + db.setCount('myTestSet', function(err, data) { + console.log('setCount return', data); + callback(err, {'setCount': data}); + }); + } + var objectTasks = [ @@ -345,8 +352,15 @@ var DebugRoute = function(app) { getSetMembers, setAdd, getSetMembers, + setRemove, + getSetMembers, + isSetMember, + setAdd, + getSetMembers, isSetMember, + setRemoveRandom, getSetMembers, + setCount, isMemberOfSets ]; From 39b70a9e09c9276eb4231c738056eac431fce57a Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 22:55:31 -0500 Subject: [PATCH 47/83] derp --- src/routes/debug.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/routes/debug.js b/src/routes/debug.js index b413e2b5cd..e431b34ed7 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -165,21 +165,21 @@ var DebugRoute = function(app) { function sortedSetAdd(callback) { - db.sortedSetAdd('sortedSet2', 1, 12, function(err, data) { + db.sortedSetAdd('sortedSet3', 12, 5, function(err, data) { console.log('sortedSetAdd return', data); callback(err, {'sortedSetAdd': data}); }); } function sortedSetRemove(callback) { - db.sortedSetRemove('sortedSet2', 12, function(err, data) { + db.sortedSetRemove('sortedSet3', 12, function(err, data) { console.log('sortedSetRemove return', data); callback(err, {'sortedSetRemove': data}); }); } function getSortedSetRange(callback) { - db.getSortedSetRevRange('sortedSet2', 0, -1, function(err, data) { + db.getSortedSetRevRange('sortedSet3', 0, -1, function(err, data) { console.log('getSortedSetRange return', data); callback(err, {'getSortedSetRange': data}); }); @@ -325,7 +325,7 @@ var DebugRoute = function(app) { getSortedSetRange, sortedSetAdd, getSortedSetRange, - sortedSetRemove, + //sortedSetRemove, getSortedSetRange, ]; @@ -364,7 +364,7 @@ var DebugRoute = function(app) { isMemberOfSets ]; - require('async').series(setTasks, function(err, results) { + require('async').series(sortedSetTasks, function(err, results) { if(err) { console.log(err); res.send(err.message); From c698af17ae3498c9a25ee2e87ce79359152d738a Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 23:36:52 -0500 Subject: [PATCH 48/83] added error checking to collection index creation --- src/database/mongo.js | 16 +++++++++++----- src/threadTools.js | 2 ++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 6e2db0c4f0..82ad29223b 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -27,11 +27,17 @@ db.createCollection('objects', function(err, collection) { - collection.ensureIndex({_key :1}, {background:true}, function(err, name){ - if(err) { - winston.error("Error creating index " + err.message); - } - }); + if(err) { + winston.error("Error creating collection " + err.message); + return; + } + if(collection) { + collection.ensureIndex({_key :1, setName:1}, {background:true}, function(err, name){ + if(err) { + winston.error("Error creating index " + err.message); + } + }); + } }); callback(err); diff --git a/src/threadTools.js b/src/threadTools.js index bd0a2a555d..a392241131 100644 --- a/src/threadTools.js +++ b/src/threadTools.js @@ -17,7 +17,9 @@ var db = require('./database'), (function(ThreadTools) { ThreadTools.exists = function(tid, callback) { + db.isSetMember('topics:tid', tid, function(err, ismember) { + if (err) { callback(false); } From 567997ef3c665bbbf5ba41523847c541db3b5749 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Wed, 4 Dec 2013 23:51:57 -0500 Subject: [PATCH 49/83] changed mongo sets to store just strings, fixes worlds problems --- src/database/mongo.js | 4 ++-- src/feed.js | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 82ad29223b..45f8814b3b 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -273,7 +273,7 @@ // sets module.setAdd = function(key, value, callback) { - db.collection('objects').update({_key:key}, {$addToSet: { members: value }}, {upsert:true, w: 1}, function(err, result) { + db.collection('objects').update({_key:key}, {$addToSet: { members: value.toString() }}, {upsert:true, w: 1}, function(err, result) { if(callback) { callback(err, result); } @@ -289,7 +289,7 @@ } module.isSetMember = function(key, value, callback) { - db.collection('objects').findOne({_key:key, members: value}, function(err, item) { + db.collection('objects').findOne({_key:key, members: value.toString()}, function(err, item) { callback(err, item !== null && item !== undefined); }); } diff --git a/src/feed.js b/src/feed.js index fab4fe05a4..798c8bdabd 100644 --- a/src/feed.js +++ b/src/feed.js @@ -30,7 +30,12 @@ Feed.updateTopic = function (tid, callback) { topics.getTopicWithPosts(tid, 0, 0, -1, function (err, topicData) { if (err) { - return callback(new Error('topic-invalid')); + if(callback) { + return callback(new Error('topic-invalid')); + } else { + winston.error(err.message); + return; + } } var feed = new rss({ From 2dd295118caa71ac9489d2140791bd933c1cd661 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Thu, 5 Dec 2013 12:04:09 -0500 Subject: [PATCH 50/83] setRemove converts to string too --- src/database/mongo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 45f8814b3b..609c6f9f3f 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -281,7 +281,7 @@ } module.setRemove = function(key, value, callback) { - db.collection('objects').update({_key:key, members: value}, {$pull : {members: value}}, function(err, result) { + db.collection('objects').update({_key:key, members: value.toString()}, {$pull : {members: value}}, function(err, result) { if(callback) { callback(err, result); } From 7875138c082bf94f76b7902ff4ac75b74f9ceaa1 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Thu, 5 Dec 2013 13:11:27 -0500 Subject: [PATCH 51/83] added parseInt --- public/src/forum/admin/settings.js | 2 +- src/database/mongo.js | 2 +- src/feed.js | 4 ++-- src/groups.js | 6 +++--- src/login.js | 2 +- src/postTools.js | 6 +++--- src/posts.js | 8 ++++---- src/routes/admin.js | 4 ++-- src/routes/api.js | 11 ++++++----- src/routes/user.js | 11 ++++++----- src/threadTools.js | 2 +- src/topics.js | 20 ++++++++++---------- src/user.js | 8 +------- src/webserver.js | 4 ++-- src/websockets.js | 6 +++--- 15 files changed, 46 insertions(+), 50 deletions(-) diff --git a/public/src/forum/admin/settings.js b/public/src/forum/admin/settings.js index 6c75e6457a..989e94a79c 100644 --- a/public/src/forum/admin/settings.js +++ b/public/src/forum/admin/settings.js @@ -32,7 +32,7 @@ define(['uploader'], function(uploader) { break; case 'checkbox': - fields[x].checked = app.config[key] === '1' ? true : false; + fields[x].checked = parseInt(app.config[key], 10) === 1; break; } } diff --git a/src/database/mongo.js b/src/database/mongo.js index 609c6f9f3f..4ba91da50e 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -369,7 +369,7 @@ }; data.setName = key; - module.setObject(key+':'+value, data, callback); + module.setObject(key + ':' + value, data, callback); } module.sortedSetRemove = function(key, value, callback) { diff --git a/src/feed.js b/src/feed.js index 798c8bdabd..d270e4b3cc 100644 --- a/src/feed.js +++ b/src/feed.js @@ -55,8 +55,8 @@ } async.each(topicData.posts, function(postData, next) { - if (postData.deleted === '0') { - dateStamp = new Date(parseInt(postData.edited === '0' ? postData.timestamp : postData.edited, 10)).toUTCString(); + if (parseInt(postData.deleted, 10) === 0) { + dateStamp = new Date(parseInt(parseInt(postData.edited, 10) === 0 ? postData.timestamp : postData.edited, 10)).toUTCString(); feed.item({ title: 'Reply to ' + topicData.topic_name + ' on ' + dateStamp, diff --git a/src/groups.js b/src/groups.js index 972cb9545f..da521a75af 100644 --- a/src/groups.js +++ b/src/groups.js @@ -15,7 +15,7 @@ }, function (err, groups) { // Remove deleted and hidden groups from this list callback(err, groups.filter(function (group) { - if (group.deleted === '1' || group.hidden === '1') { + if (parseInt(group.deleted, 10) === 1 || parseInt(group.hidden, 10) === 1) { return false; } else { return true; @@ -76,7 +76,7 @@ Groups.isDeleted = function(gid, callback) { db.getObjectField('gid:' + gid, 'deleted', function(err, deleted) { - callback(err, deleted === '1'); + callback(err, parseInt(deleted, 10) === 1); }); }; @@ -240,7 +240,7 @@ return next(err); } - if (groupObj.deleted === '1') { + if (parseInt(groupObj.deleted, 10) === 1) { db.deleteObjectField('group:gid', groupObj.name, function(err) { db.delete('gid:' + gid, function(err) { diff --git a/src/login.js b/src/login.js index adce1f8bfc..c4eb226586 100644 --- a/src/login.js +++ b/src/login.js @@ -28,7 +28,7 @@ var user = require('./user'), user.getUserFields(uid, ['password', 'banned'], function(err, userData) { if (err) return next(err); - if (userData.banned && userData.banned === '1') { + if (userData.banned && parseInt(userData.banned, 10) === 1) { return next({ status: "error", message: "user-banned" diff --git a/src/postTools.js b/src/postTools.js index 2641b8d9b2..0ec6abc81a 100644 --- a/src/postTools.js +++ b/src/postTools.js @@ -164,7 +164,7 @@ var db = require('./database'), }; posts.getPostField(pid, 'deleted', function(err, deleted) { - if(deleted === '1') { + if(parseInt(deleted, 10) === 1) { return callback(new Error('Post already deleted!')); } @@ -195,7 +195,7 @@ var db = require('./database'), // Restore topic if it is the only post topics.getTopicField(postData.tid, 'postcount', function(err, count) { - if (count === '1') { + if (parseInt(count, 10) === 1) { threadTools.restore(postData.tid, uid); } }); @@ -210,7 +210,7 @@ var db = require('./database'), }; posts.getPostField(pid, 'deleted', function(err, deleted) { - if(deleted === '0') { + if(parseInt(deleted, 10) === 0) { return callback(new Error('Post already restored')); } diff --git a/src/posts.js b/src/posts.js index 8dc8900f97..f6375d2401 100644 --- a/src/posts.js +++ b/src/posts.js @@ -78,7 +78,7 @@ var db = require('./database'), db.sortedSetAdd('categories:recent_posts:cid:' + cid, timestamp, pid); - if(topicData.pinned === '0') { + if(parseInt(topicData.pinned, 10) === 0) { db.sortedSetAdd('categories:' + cid + ':tid', timestamp, tid); } @@ -213,7 +213,7 @@ var db = require('./database'), post.userslug = userData.userslug || ''; post.user_rep = userData.reputation || 0; post.user_postcount = userData.postcount || 0; - post.user_banned = userData.banned === '1'; + post.user_banned = parseInt(userData.banned, 10) === 1; post.picture = userData.picture || require('gravatar').url('', {}, https = nconf.get('https')); post.signature = signature; @@ -250,7 +250,7 @@ var db = require('./database'), async.waterfall([ function(next) { Posts.getPostFields(pid, ['pid', 'tid', 'content', 'uid', 'timestamp', 'deleted'], function(err, postData) { - if (postData.deleted === '1') { + if (parseInt(postData.deleted, 10) === 1) { return callback(null); } else { postData.relativeTime = new Date(parseInt(postData.timestamp || 0, 10)).toISOString(); @@ -267,7 +267,7 @@ var db = require('./database'), topics.getTopicFields(postData.tid, ['title', 'cid', 'slug', 'deleted'], function(err, topicData) { if (err) { return callback(err); - } else if (topicData.deleted === '1') { + } else if (parseInt(topicData.deleted, 10) === 1) { return callback(null); } categories.getCategoryFields(topicData.cid, ['name', 'icon', 'slug'], function(err, categoryData) { diff --git a/src/routes/admin.js b/src/routes/admin.js index 666877560c..e8402442f9 100644 --- a/src/routes/admin.js +++ b/src/routes/admin.js @@ -241,7 +241,7 @@ var nconf = require('nconf'), app.get('/categories/active', function (req, res) { categories.getAllCategories(0, function (err, data) { data.categories = data.categories.filter(function (category) { - return (!category.disabled || category.disabled === "0"); + return (!category.disabled || parseInt(category.disabled, 10) === 0); }); res.json(data); }); @@ -250,7 +250,7 @@ var nconf = require('nconf'), app.get('/categories/disabled', function (req, res) { categories.getAllCategories(0, function (err, data) { data.categories = data.categories.filter(function (category) { - return category.disabled === "1"; + return parseInt(category.disabled, 10) === 1; }); res.json(data); }); diff --git a/src/routes/api.js b/src/routes/api.js index 389763bac5..3510f5bdec 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -42,7 +42,7 @@ var path = require('path'), var uid = (req.user) ? req.user.uid : 0; categories.getAllCategories(uid, function (err, data) { data.categories = data.categories.filter(function (category) { - return (!category.disabled || category.disabled === "0"); + return (!category.disabled || parseInt(category.disabled, 10) === 0); }); function iterator(category, callback) { @@ -54,7 +54,7 @@ var path = require('path'), } async.each(data.categories, iterator, function (err) { - data.motd_class = (meta.config.show_motd === '1' || meta.config.show_motd === undefined) ? '' : ' none'; + data.motd_class = (parseInt(meta.config.show_motd, 10) === 1 || meta.config.show_motd === undefined) ? '' : ' none'; data.motd_class += (meta.config.motd && meta.config.motd.length > 0 ? '' : ' default'); data.motd = require('marked')(meta.config.motd || "
 Get NodeBB  Fork us on Github  @dcplabs
\n\n# NodeBB v" + pkg.version + "\nWelcome to NodeBB, the discussion platform of the future."); @@ -117,7 +117,7 @@ var path = require('path'), var uid = (req.user) ? req.user.uid : 0; topics.getTopicWithPosts(req.params.id, uid, 0, 10, function (err, data) { if (!err) { - if (data.deleted === '1' && data.expose_tools === 0) { + if (parseInt(data.deleted, 10) === 1 && parseInt(data.expose_tools, 10) === 0) { return res.json(404, {}); } res.json(data); @@ -132,10 +132,11 @@ var path = require('path'), categoryTools.privileges(req.params.id, uid, function(err, privileges) { if (!err && privileges.read) { categories.getCategoryById(req.params.id, uid, function (err, data) { - if (!err && data && data.disabled === "0") + if (!err && data && parseInt(data.disabled, 10) === 0) { res.json(data); - else + } else { next(); + } }, req.params.id, uid); } else { res.send(403); diff --git a/src/routes/user.js b/src/routes/user.js index 58231a16c6..80f97347a1 100644 --- a/src/routes/user.js +++ b/src/routes/user.js @@ -318,10 +318,11 @@ var fs = require('fs'), return next(err); if (userData) { - if (userData.showemail && userData.showemail === "1") + if (userData.showemail && parseInt(userData.showemail, 10) === 1) { userData.showemail = "checked"; - else + } else { userData.showemail = ""; + } res.json(userData); } else { res.json(404, { @@ -501,21 +502,21 @@ var fs = require('fs'), } function canSeeEmail() { - return callerUID == uid || (data.email && (data.showemail && data.showemail === "1")); + return callerUID == uid || (data.email && (data.showemail && parseInt(data.showemail, 10) === 1)); } if (!canSeeEmail()) { data.email = ""; } - if (callerUID == uid && (!data.showemail || data.showemail === "0")) { + if (callerUID == uid && (!data.showemail || parseInt(data.showemail, 10) === 0)) { data.emailClass = ""; } else { data.emailClass = "hide"; } data.websiteName = data.website.replace('http://', '').replace('https://', ''); - data.banned = data.banned === '1'; + data.banned = parseInt(data.banned, 10) === 1; data.uid = uid; data.yourid = callerUID; data.theirid = uid; diff --git a/src/threadTools.js b/src/threadTools.js index a392241131..939603958c 100644 --- a/src/threadTools.js +++ b/src/threadTools.js @@ -292,7 +292,7 @@ var db = require('./database'), pids.reverse(); async.detectSeries(pids, function(pid, next) { posts.getPostField(pid, 'deleted', function(err, deleted) { - if (deleted === '0') { + if (parseInt(deleted, 10) === 0) { next(true); } else { next(false); diff --git a/src/topics.js b/src/topics.js index 4345545685..fc80cc965b 100644 --- a/src/topics.js +++ b/src/topics.js @@ -170,7 +170,7 @@ var async = require('async'), } postData = postData.filter(function(post) { - return parseInt(current_user, 10) !== 0 || post.deleted === "0"; + return parseInt(current_user, 10) !== 0 || parseInt(post.deleted, 10) === 0; }); function getFavouritesData(next) { @@ -454,18 +454,18 @@ var async = require('async'), getTopicInfo(topicData, function(topicInfo) { - topicData['pin-icon'] = topicData.pinned === '1' ? 'fa-thumb-tack' : 'none'; - topicData['lock-icon'] = topicData.locked === '1' ? 'fa-lock' : 'none'; - topicData['deleted-class'] = topicData.deleted === '1' ? 'deleted' : ''; + topicData['pin-icon'] = parseInt(topicData.pinned, 10) === 1 ? 'fa-thumb-tack' : 'none'; + topicData['lock-icon'] = parseInt(topicData.locked, 10) === 1 ? 'fa-lock' : 'none'; + topicData['deleted-class'] = parseInt(topicData.deleted, 10) === 1 ? 'deleted' : ''; - topicData.unreplied = topicData.postcount === '1'; + topicData.unreplied = parseInt(topicData.postcount, 10) === 1; topicData.username = topicInfo.username || 'anonymous'; topicData.userslug = topicInfo.userslug || ''; topicData.picture = topicInfo.picture || gravatar.url('', {}, https = nconf.get('https')); topicData.categoryIcon = topicInfo.categoryData.icon; topicData.categoryName = topicInfo.categoryData.name; topicData.categorySlug = topicInfo.categoryData.slug; - topicData.badgeclass = (topicInfo.hasread && current_user != 0) ? '' : 'badge-important'; + topicData.badgeclass = (topicInfo.hasread && parseInt(current_user, 10) !== 0) ? '' : 'badge-important'; topicData.teaser_text = topicInfo.teaserInfo.text || '', topicData.teaser_username = topicInfo.teaserInfo.username || ''; topicData.teaser_userslug = topicInfo.teaserInfo.userslug || ''; @@ -555,7 +555,7 @@ var async = require('async'), } function getReadStatus(next) { - if (uid && parseInt(uid) > 0) { + if (uid && parseInt(uid, 10) > 0) { Topics.hasReadTopic(tid, uid, function(read) { next(null, read); }); @@ -580,8 +580,8 @@ var async = require('async'), hasRead = results[1], teaser = results[2]; - topicData['pin-icon'] = topicData.pinned === '1' ? 'fa-thumb-tack' : 'none'; - topicData['lock-icon'] = topicData.locked === '1' ? 'fa-lock' : 'none'; + topicData['pin-icon'] = parseInt(topicData.pinned, 10) === 1 ? 'fa-thumb-tack' : 'none'; + topicData['lock-icon'] = parseInt(topicData.locked, 10) === 1 ? 'fa-lock' : 'none'; topicData.badgeclass = hasRead ? '' : 'badge-important'; topicData.teaser_text = teaser.text || ''; @@ -808,7 +808,7 @@ var async = require('async'), if(err) { return callback(err, null); } - callback(null, locked === "1"); + callback(null, parseInt(locked, 10) === 1); }); } diff --git a/src/user.js b/src/user.js index 29f574174d..8bc99bea91 100644 --- a/src/user.js +++ b/src/user.js @@ -182,12 +182,6 @@ var bcrypt = require('bcrypt'), }); }; - User.filterBannedUsers = function(users) { - return users.filter(function(user) { - return (!user.banned || user.banned === '0'); - }); - }; - User.updateProfile = function(uid, data, callback) { var fields = ['email', 'fullname', 'website', 'location', 'birthday', 'signature']; @@ -550,7 +544,7 @@ var bcrypt = require('bcrypt'), } function iterator(uid, callback) { - if(uid === "0") { + if(parseInt(uid, 10) === 0) { return callback(null); } diff --git a/src/webserver.js b/src/webserver.js index e98f3fcfe3..0c224f6569 100644 --- a/src/webserver.js +++ b/src/webserver.js @@ -465,7 +465,7 @@ var path = require('path'), function (next) { topics.getTopicWithPosts(tid, ((req.user) ? req.user.uid : 0), 0, -1, function (err, topicData) { if (topicData) { - if (topicData.deleted === '1' && topicData.expose_tools === 0) { + if (parseInt(topicData.deleted, 10) === 1 && parseInt(topicData.expose_tools, 10) === 0) { return next(new Error('Topic deleted'), null); } } @@ -587,7 +587,7 @@ var path = require('path'), categories.getCategoryById(cid, 0, function (err, categoryData) { if (categoryData) { - if (categoryData.disabled === '1') { + if (parseInt(categoryData.disabled, 10) === 1) { return next(new Error('Category disabled'), null); } } diff --git a/src/websockets.js b/src/websockets.js index 09667ca401..0eb93febbb 100644 --- a/src/websockets.js +++ b/src/websockets.js @@ -349,7 +349,7 @@ websockets.init = function(io) { }); socket.on('api:topics.post', function(data) { - if (uid < 1 && meta.config.allowGuestPosting === '0') { + if (uid < 1 && parseInt(meta.config.allowGuestPosting, 10) === 0) { socket.emit('event:alert', { title: 'Post Unsuccessful', message: 'You don't seem to be logged in, so you cannot reply.', @@ -420,7 +420,7 @@ websockets.init = function(io) { }); socket.on('api:posts.reply', function(data) { - if (uid < 1 && meta.config.allowGuestPosting === '0') { + if (uid < 1 && parseInt(meta.config.allowGuestPosting, 10) === 0) { socket.emit('event:alert', { title: 'Reply Unsuccessful', message: 'You don't seem to be logged in, so you cannot reply.', @@ -772,7 +772,7 @@ websockets.init = function(io) { }); socket.on('api:composer.push', function(data) { - if (uid > 0 || meta.config.allowGuestPosting === '1') { + if (parseInt(uid, 10) > 0 || parseInt(meta.config.allowGuestPosting, 10) === 1) { if (parseInt(data.tid) > 0) { topics.getTopicData(data.tid, function(err, topicData) { if (data.body) From 67d5ea83e74221f9e68e237a2e7121cf5e137efe Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Thu, 5 Dec 2013 13:33:01 -0500 Subject: [PATCH 52/83] replace . with \uff0e in mongo --- src/database/mongo.js | 7 ++++++- src/routes/debug.js | 11 ++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 4ba91da50e..4b577cf9c6 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -148,6 +148,8 @@ module.setObjectField = function(key, field, value, callback) { var data = {}; + // if there is a '.' in the field name it inserts subdocument in mongo, replace '.'s with \uff0E + field = field.replace(/\./g, '\uff0E'); data[field] = value; db.collection('objects').update({_key:key}, {$set:data}, {upsert:true, w: 1}, function(err, result) { if(callback) { @@ -185,7 +187,7 @@ var _fields = {}; for(var i=0; i Date: Thu, 5 Dec 2013 14:24:18 -0500 Subject: [PATCH 53/83] added mongo info function, change admin redis template to database template --- .../admin/{redis.tpl => database.tpl} | 18 ++++++++++++++++++ public/templates/admin/header.tpl | 2 +- public/templates/config.json | 2 +- src/database/mongo.js | 10 +++++++++- src/database/redis.js | 10 +++++++--- src/routes/admin.js | 4 ++-- 6 files changed, 38 insertions(+), 8 deletions(-) rename public/templates/admin/{redis.tpl => database.tpl} (67%) diff --git a/public/templates/admin/redis.tpl b/public/templates/admin/database.tpl similarity index 67% rename from public/templates/admin/redis.tpl rename to public/templates/admin/database.tpl index aa127ef893..0176d4d697 100644 --- a/public/templates/admin/redis.tpl +++ b/public/templates/admin/database.tpl @@ -1,3 +1,6 @@ + + +

Redis

@@ -21,4 +24,19 @@ Keyspace Hits {keyspace_hits}
Keyspace Misses {keyspace_misses}
+ + +

Mongo

+
+
+ + Collections {collections}
+ Objects {objects}
+ Avg. Object Size {avgObjSize} kb
+
+ Data Size {dataSize} kb
+ Storage Size {storageSize} kb
+ File Size {fileSize} kb
+
+ diff --git a/public/templates/admin/header.tpl b/public/templates/admin/header.tpl index 786b8bcfaa..66ef0fe982 100644 --- a/public/templates/admin/header.tpl +++ b/public/templates/admin/header.tpl @@ -105,7 +105,7 @@
  • Themes
  • Plugins
  • Settings
    • -
  • Redis
    • +
  • Database
  • Logger
  • MOTD
    • diff --git a/public/templates/config.json b/public/templates/config.json index fa435245ae..45e922c3df 100644 --- a/public/templates/config.json +++ b/public/templates/config.json @@ -4,7 +4,7 @@ "^admin/topics.*": "admin/topics", "^admin/categories.*": "admin/categories", "^admin/users.*": "admin/users", - "^admin/redis.*": "admin/redis", + "^admin/database.*": "admin/database", "^admin/index.*": "admin/index", "^admin/themes.*": "admin/themes", "^admin/plugins/?$": "admin/plugins", diff --git a/src/database/mongo.js b/src/database/mongo.js index 4b577cf9c6..d41ccfddaa 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -71,7 +71,15 @@ } module.info = function(callback) { - throw new Error('not-implemented'); + db.stats({scale:1024}, function(err, stats) { + + stats.avgObjSize = (stats.avgObjSize / 1024).toFixed(2); + + stats.mongo = true; + //remove this when andrew adds in undefined checking to templates + stats.redis = false; + callback(err, stats); + }); } // key diff --git a/src/database/redis.js b/src/database/redis.js index cfb34dad10..14eebdcb3a 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -103,7 +103,7 @@ } data = data.split("\r\n"); - var finalData = {}; + var redisData = {}; for (var i in data) { @@ -116,14 +116,18 @@ var jsonObject = JSON.parse(json); for (var key in jsonObject) { - finalData[key] = jsonObject[key]; + redisData[key] = jsonObject[key]; } } catch (err) { winston.warn('can\'t parse redis status variable, ignoring', i, data[i], err); } } - callback(null, finalData); + redisData.redis = true; + //remove this when andrew adds in undefined checking to templates + redisData.mongo = false; + + callback(null, redisData); }); } diff --git a/src/routes/admin.js b/src/routes/admin.js index e8402442f9..2831787434 100644 --- a/src/routes/admin.js +++ b/src/routes/admin.js @@ -55,7 +55,7 @@ var nconf = require('nconf'), (function () { var routes = [ 'categories/active', 'categories/disabled', 'users', 'topics', 'settings', 'themes', - 'twitter', 'facebook', 'gplus', 'redis', 'motd', 'groups', 'plugins', 'logger', + 'twitter', 'facebook', 'gplus', 'database', 'motd', 'groups', 'plugins', 'logger', 'users/latest', 'users/sort-posts', 'users/sort-reputation', 'users/search' ]; @@ -265,7 +265,7 @@ var nconf = require('nconf'), }); }); - app.namespace('/redis', function () { + app.namespace('/database', function () { app.get('/', function (req, res) { db.info(function (err, data) { res.json(data); From b927f6ce2938f815021e57a101d6740c9af5b092 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Thu, 5 Dec 2013 14:30:18 -0500 Subject: [PATCH 54/83] added raw info --- public/templates/admin/database.tpl | 10 ++++++++++ src/database/mongo.js | 2 ++ src/database/redis.js | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/public/templates/admin/database.tpl b/public/templates/admin/database.tpl index 0176d4d697..40feb65c12 100644 --- a/public/templates/admin/database.tpl +++ b/public/templates/admin/database.tpl @@ -24,6 +24,11 @@ Keyspace Hits {keyspace_hits}
    Keyspace Misses {keyspace_misses}
    +
    +

    Raw Info

    +
    +
    {raw}
    +
    @@ -39,4 +44,9 @@ Storage Size {storageSize} kb
    File Size {fileSize} kb
    +
    +

    Raw Info

    +
    +
    {raw}
    +
    diff --git a/src/database/mongo.js b/src/database/mongo.js index d41ccfddaa..c90f6bc3cf 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -75,6 +75,8 @@ stats.avgObjSize = (stats.avgObjSize / 1024).toFixed(2); + stats.raw = JSON.stringify(stats, null, 4); + stats.mongo = true; //remove this when andrew adds in undefined checking to templates stats.redis = false; diff --git a/src/database/redis.js b/src/database/redis.js index 14eebdcb3a..ca7f3483f2 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -122,7 +122,7 @@ winston.warn('can\'t parse redis status variable, ignoring', i, data[i], err); } } - + redisData.raw = JSON.stringify(redisData, null, 4); redisData.redis = true; //remove this when andrew adds in undefined checking to templates redisData.mongo = false; From fe527ff2a90d2670372b3678e10ef24474ffee20 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Thu, 5 Dec 2013 14:48:58 -0500 Subject: [PATCH 55/83] removed RedisStoreLib from websockets.js it moved into redis.js --- src/websockets.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/websockets.js b/src/websockets.js index 0eb93febbb..43d9840de0 100644 --- a/src/websockets.js +++ b/src/websockets.js @@ -9,7 +9,6 @@ var cookie = require('cookie'), gravatar = require('gravatar'), winston = require('winston'), - RedisStoreLib = require('connect-redis')(express), db = require('./database'), user = require('./user'), From 806a454b056e75a74cda0eb1ac61526b50cb468c Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Thu, 5 Dec 2013 15:25:58 -0500 Subject: [PATCH 56/83] fixing notifications.prune --- src/database/mongo.js | 9 +++++- src/database/redis.js | 14 +++++++++ src/notifications.js | 73 +++++++++++++++++++++++++------------------ src/routes/debug.js | 12 +++++++ 4 files changed, 76 insertions(+), 32 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index c90f6bc3cf..a9e17b8e02 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -20,7 +20,6 @@ process.exit(); } - // TODO: fill out settings.db module.sessionStore = new mongoStore({ db: db }); @@ -477,6 +476,14 @@ }); } + module.sortedSetScore = function(key, value, callback) { + throw new Error('not-implemented'); + } + + module.sortedSetsScore = function(keys, value, callback) { + throw new Error('not-implemented'); + } + // lists module.listPrepend = function(key, value, callback) { module.isObjectField(key, 'array', function(err, exists) { diff --git a/src/database/redis.js b/src/database/redis.js index ca7f3483f2..b2a17845ce 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -311,6 +311,20 @@ redisClient.zrank(key, value, callback); } + module.sortedSetScore = function(key, value, callback) { + redisClient.zscore(key, value, callback); + } + + module.sortedSetsScore = function(keys, value, callback) { + var multi = redisClient.multi(); + + for(x=0; x Date: Thu, 5 Dec 2013 15:26:30 -0500 Subject: [PATCH 57/83] more --- src/notifications.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/notifications.js b/src/notifications.js index 47cf782368..874e39bc3d 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -193,7 +193,6 @@ var async = require('async'), Notifications.prune = function(cutoff) { // TODO: this function wont work with dbal in its current state // things to figure out - // 1 - RDB.keys mongo uses regex, redis uses global patterns? // 2 - Need to remove the RDB.multi if (process.env.NODE_ENV === 'development') { @@ -211,7 +210,6 @@ var async = require('async'), async.parallel({ "inboxes": function(next) { - //RDB.keys('uid:*:notifications:unread', next); db.getSortedSetRange('users:joindate', 0, -1, function(err, uids) { if(err) { return next(err); @@ -248,7 +246,7 @@ var async = require('async'), var numInboxes = results.inboxes.length, x; - console.log(results.inboxes, results.expiredNids); + async.eachSeries(results.expiredNids, function(nid, next) { var multi = RDB.multi(); From 58df656c65980b4d2fcb3937039f45e1389e8079 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Thu, 5 Dec 2013 15:27:14 -0500 Subject: [PATCH 58/83] removed temp vars --- src/notifications.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/notifications.js b/src/notifications.js index 874e39bc3d..ada2a586b1 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -244,13 +244,10 @@ var async = require('async'), return; } - var numInboxes = results.inboxes.length, - x; - async.eachSeries(results.expiredNids, function(nid, next) { var multi = RDB.multi(); - for(x=0; x Date: Thu, 5 Dec 2013 15:29:27 -0500 Subject: [PATCH 59/83] added db function --- src/notifications.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/notifications.js b/src/notifications.js index ada2a586b1..8e598bb635 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -245,13 +245,11 @@ var async = require('async'), } async.eachSeries(results.expiredNids, function(nid, next) { - var multi = RDB.multi(); - for(var x=0; x Date: Thu, 5 Dec 2013 15:32:36 -0500 Subject: [PATCH 60/83] more notif.prune cleanup --- src/notifications.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/notifications.js b/src/notifications.js index 8e598bb635..9c4796858c 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -191,9 +191,6 @@ var async = require('async'), }; Notifications.prune = function(cutoff) { - // TODO: this function wont work with dbal in its current state - // things to figure out - // 2 - Need to remove the RDB.multi if (process.env.NODE_ENV === 'development') { winston.info('[notifications.prune] Removing expired notifications from the database.'); @@ -250,11 +247,10 @@ var async = require('async'), if(err) { return next(err); } + // If the notification is not present in any inbox, delete it altogether var expired = results.every(function(present) { - if (present === null) { - return true; - } + return present === null; }); if (expired) { From 1564e3d53048b719c873ed591fc57e1f20aabdb5 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 18:26:26 -0500 Subject: [PATCH 61/83] tons of fixes to mongo and redis, to make returns the same --- src/database/mongo.js | 78 ++++++++++++++++++++++++++++++++++--------- src/database/redis.js | 2 +- src/routes/debug.js | 33 ++++++++++++++---- src/topics.js | 2 +- 4 files changed, 92 insertions(+), 23 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index a9e17b8e02..d5b18b85dd 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -86,7 +86,7 @@ // key module.exists = function(key, callback) { - db.collection('objects').findOne({_key:key}, function(err, item) { + db.collection('objects').findOne({$or:[{_key:key}, {setName:key}]}, function(err, item) { callback(err, item !== undefined && item !== null); }); } @@ -176,9 +176,27 @@ } module.getObjects = function(keys, callback) { - db.collection('objects').find({_key:{$in:keys}}, {_id:0, _key:0}).toArray(function(err, data) { + db.collection('objects').find({_key:{$in:keys}}, {_id:0}).toArray(function(err, data) { + + if(err) { + return callback(err); + } + + var returnData = [], + resultIndex = 0; + + for(var i=0; i Date: Thu, 5 Dec 2013 18:31:15 -0500 Subject: [PATCH 62/83] moved reds into redis, fixed notifications prune param --- src/database/redis.js | 5 +++++ src/notifications.js | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/database/redis.js b/src/database/redis.js index 9bc329d85f..76fa920c84 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -8,6 +8,7 @@ nconf = require('nconf'), express = require('express'), connectRedis = require('connect-redis')(express), + reds = require('reds'), redis_socket_or_host = nconf.get('redis:host'), utils = require('./../../public/src/utils.js'); @@ -28,6 +29,10 @@ ttl: 60 * 60 * 24 * 30 }); + reds.createClient = function () { + return reds.client || (reds.client = redisClient); + }; + if (nconf.get('redis:password')) { redisClient.auth(nconf.get('redis:password')); } diff --git a/src/notifications.js b/src/notifications.js index 9c4796858c..030478701b 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -243,7 +243,7 @@ var async = require('async'), async.eachSeries(results.expiredNids, function(nid, next) { - db.sortedSetsScore(results.inboxes, function(err, results) { + db.sortedSetsScore(results.inboxes, nid, function(err, results) { if(err) { return next(err); } From 6c70d37f1c4881d25fd9690d31ba0d3f1f36d2e1 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 18:32:45 -0500 Subject: [PATCH 63/83] removed reds from app.js --- app.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/app.js b/app.js index 9c839a94d0..103a2d19e1 100644 --- a/app.js +++ b/app.js @@ -76,15 +76,6 @@ require('./src/database').init(function(err) { meta.configs.init(function () { - // - // TODO : figure out reds search after dbal is complete - // - //var reds = require('reds'), - // db = require('./src/database'); - /*reds.createClient = function () { - return reds.client || (reds.client = db); - };*/ - var templates = require('./public/src/templates'), translator = require('./public/src/translator'), webserver = require('./src/webserver'), From 53ca7a114369a99916ab9265d7a86e69559d293f Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 20:06:36 -0500 Subject: [PATCH 64/83] added search functions to database files, removed reds from nodebb moved it to redis --- src/database/mongo.js | 54 ++++++++++++++++++++++++++++++++++++++----- src/database/redis.js | 39 +++++++++++++++++++++++++++++++ src/postTools.js | 16 ++++++------- src/posts.js | 8 +++---- src/routes/api.js | 16 +++---------- src/routes/debug.js | 4 +++- src/threadTools.js | 7 ++---- src/topics.js | 4 +--- src/user.js | 9 ++++---- 9 files changed, 110 insertions(+), 47 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index d5b18b85dd..699c46af6f 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -39,6 +39,20 @@ } }); + db.createCollection('search', function(err, collection) { + if(err) { + winston.error("Error creating collection " + err.message); + return; + } + if(collection) { + collection.ensureIndex({content:'text'}, {background:true}, function(err, name){ + if(err) { + winston.error("Error creating index " + err.message); + } + }); + } + }) + callback(err); }); @@ -54,6 +68,28 @@ // Exported functions // + module.searchIndex = function(key, content, id) { + + db.collection('search').update({key:key, content:content, _id:id}, {upsert:true, w: 1}, function(err, result) { + + }); + } + + module.search = function(key, term, callback) { + db.command({text:"search" , search: term }, function(err, result) { + callback(err, result) + }); + /*db.runCommand("text", { search: term }, function(err, result) { + callback(err, result); + });*/ + } + + module.searchRemove = function(key, id) { + db.collection('search').remove({_id:id}, function(err, result) { + callback(err, result); + }); + } + module.flushdb = function(callback) { db.dropDatabase(function(err, result) { if(err){ @@ -72,14 +108,20 @@ module.info = function(callback) { db.stats({scale:1024}, function(err, stats) { - stats.avgObjSize = (stats.avgObjSize / 1024).toFixed(2); + db.serverStatus(function(err, serverStatus) { + + stats.avgObjSize = (stats.avgObjSize / 1024).toFixed(2); - stats.raw = JSON.stringify(stats, null, 4); + stats.serverStatus = serverStatus; - stats.mongo = true; - //remove this when andrew adds in undefined checking to templates - stats.redis = false; - callback(err, stats); + stats.raw = JSON.stringify(stats, null, 4); + + stats.mongo = true; + //remove this when andrew adds in undefined checking to templates + stats.redis = false; + callback(err, stats); + + }); }); } diff --git a/src/database/redis.js b/src/database/redis.js index 76fa920c84..7361be3458 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -33,6 +33,10 @@ return reds.client || (reds.client = redisClient); }; + var userSearch = reds.createSearch('nodebbusersearch'), + postSearch = reds.createSearch('nodebbpostsearch'), + topicSearch = reds.createSearch('nodebbtopicsearch'); + if (nconf.get('redis:password')) { redisClient.auth(nconf.get('redis:password')); } @@ -69,6 +73,41 @@ // // Exported functions // + module.searchIndex = function(key, content, id) { + if(key === 'post') { + postSearch.index(content, id); + } else if(key === 'topic') { + topicSearch.index(content, id); + } else if(key === 'user') { + userSearch.index(content, id); + } + } + + module.search = function(key, term, callback) { + function search(searchObj, callback) { + searchObj + .query(query = term).type('or') + .end(callback); + } + + if(key === 'post') { + search(postSearch, callback); + } else if(key === 'topic') { + search(topicSearch, callback); + } else if(key === 'user') { + search(userSearch, callback); + } + } + + module.searchRemove = function(key, id) { + if(key === 'post') { + postSearch.remove(id); + } else if(key === 'topic') { + topicSearch.remove(id); + } else if(key === 'user') { + userSearch.remove(id); + } + } module.flushdb = function(callback) { redisClient.send_command('flushdb', [], function(err) { diff --git a/src/postTools.js b/src/postTools.js index 0ec6abc81a..e31fa70552 100644 --- a/src/postTools.js +++ b/src/postTools.js @@ -10,9 +10,7 @@ var db = require('./database'), utils = require('../public/src/utils'), plugins = require('./plugins'), - reds = require('reds'), - postSearch = reds.createSearch('nodebbpostsearch'), - topicSearch = reds.createSearch('nodebbtopicsearch'), + winston = require('winston'), meta = require('./meta'), Feed = require('./feed'); @@ -83,8 +81,8 @@ var db = require('./database'), } ]); - postSearch.remove(pid, function() { - postSearch.index(content, pid); + db.searchRemove('post', pid, function() { + db.searchIndex('post', content, pid); }); async.parallel([ @@ -93,8 +91,8 @@ var db = require('./database'), PostTools.isMain(pid, tid, function(err, isMainPost) { if (isMainPost) { topics.setTopicField(tid, 'title', title); - topicSearch.remove(tid, function() { - topicSearch.index(title, tid); + db.searchRemove('topic', tid, function() { + db.searchIndex('topic', title, tid); }); } @@ -132,7 +130,7 @@ var db = require('./database'), var success = function() { posts.setPostField(pid, 'deleted', 1); db.decrObjectField('global', 'postCount'); - postSearch.remove(pid); + db.searchRemove('post', pid); posts.getPostFields(pid, ['tid', 'uid'], function(err, postData) { db.incrObjectFieldBy('topic:' + postData.tid, 'postcount', -1); @@ -203,7 +201,7 @@ var db = require('./database'), Feed.updateTopic(postData.tid); Feed.updateRecent(); - postSearch.index(postData.content, pid); + db.searchIndex('post', postData.content, pid); callback(); }); diff --git a/src/posts.js b/src/posts.js index 255707902a..1b72549454 100644 --- a/src/posts.js +++ b/src/posts.js @@ -12,8 +12,6 @@ var db = require('./database'), meta = require('./meta'), async = require('async'), - reds = require('reds'), - postSearch = reds.createSearch('nodebbpostsearch'), nconf = require('nconf'), validator = require('validator'), winston = require('winston'); @@ -108,7 +106,7 @@ var db = require('./database'), plugins.fireHook('action:post.save', postData); - postSearch.index(content, pid); + db.searchIndex('post', content, pid); callback(null, postData); }); @@ -481,10 +479,10 @@ var db = require('./database'), function reIndex(pid, callback) { Posts.getPostField(pid, 'content', function(err, content) { - postSearch.remove(pid, function() { + db.searchRemove('post', pid, function() { if (content && content.length) { - postSearch.index(content, pid); + db.searchIndex('post', content, pid); } callback(null); }); diff --git a/src/routes/api.js b/src/routes/api.js index b92a29e1f1..0bebd9f975 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -226,18 +226,8 @@ var path = require('path'), app.get('/search/:term', function (req, res, next) { - var reds = require('reds'); - var postSearch = reds.createSearch('nodebbpostsearch'); - var topicSearch = reds.createSearch('nodebbtopicsearch'); - - function search(searchObj, callback) { - searchObj - .query(query = req.params.term).type('or') - .end(callback); - } - function searchPosts(callback) { - search(postSearch, function (err, pids) { + db.search('post', req.params.term, function(err, pids) { if (err) { return callback(err, null); } @@ -248,11 +238,11 @@ var path = require('path'), } callback(null, posts); }); - }) + }); } function searchTopics(callback) { - search(topicSearch, function (err, tids) { + db.search('topic', req.params.term, function(err, tids) { if (err) { return callback(err, null); } diff --git a/src/routes/debug.js b/src/routes/debug.js index a7a3033b54..59838d83b1 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -407,9 +407,11 @@ var DebugRoute = function(app) { var miscTests = [ function(next) { - db.sortedSetScore('users:joindate', 1, function(err, result) { + //db.searchIndex('post', 'here is content tomato testing purple orange', 1); + db.search('post', 'tomato', function(err, result) { next(err, result); }) + } ]; diff --git a/src/threadTools.js b/src/threadTools.js index 939603958c..ed32f795be 100644 --- a/src/threadTools.js +++ b/src/threadTools.js @@ -8,9 +8,6 @@ var db = require('./database'), posts = require('./posts'), meta = require('./meta'), websockets = require('./websockets'); - - reds = require('reds'), - topicSearch = reds.createSearch('nodebbtopicsearch'), winston = require('winston'), nconf = require('nconf'), @@ -94,7 +91,7 @@ var db = require('./database'), ThreadTools.lock(tid); - topicSearch.remove(tid); + db.searchRemove('topic', tid); websockets.in('topic_' + tid).emit('event:topic_deleted', { tid: tid, @@ -117,7 +114,7 @@ var db = require('./database'), }); topics.getTopicField(tid, 'title', function(err, title) { - topicSearch.index(title, tid); + db.searchIndex('topic', title, tid); }); if(callback) { diff --git a/src/topics.js b/src/topics.js index df33bd8381..9d055556ad 100644 --- a/src/topics.js +++ b/src/topics.js @@ -2,8 +2,6 @@ var async = require('async'), gravatar = require('gravatar'), nconf = require('nconf'), validator = require('validator'), - reds = require('reds'), - topicSearch = reds.createSearch('nodebbtopicsearch'), db = require('./database'), posts = require('./posts'), @@ -84,7 +82,7 @@ var async = require('async'), 'pinned': 0 }); - topicSearch.index(title, tid); + db.searchIndex('topic', title, tid); user.addTopicIdToUser(uid, tid); diff --git a/src/user.js b/src/user.js index 8bc99bea91..c4459be153 100644 --- a/src/user.js +++ b/src/user.js @@ -4,7 +4,6 @@ var bcrypt = require('bcrypt'), nconf = require('nconf'), winston = require('winston'), gravatar = require('gravatar'), - userSearch = require('reds').createSearch('nodebbusersearch'), check = require('validator').check, sanitize = require('validator').sanitize, @@ -113,7 +112,7 @@ var bcrypt = require('bcrypt'), db.sortedSetAdd('users:postcount', 0, uid); db.sortedSetAdd('users:reputation', 0, uid); - userSearch.index(username, uid); + db.searchIndex('user', username, uid); if (password !== undefined) { User.hashPassword(password, function(err, hash) { @@ -390,8 +389,8 @@ var bcrypt = require('bcrypt'), } function reIndexUser(uid, username) { - userSearch.remove(uid, function() { - userSearch.index(username, uid); + db.searchRemove('user', uid, function() { + db.searchIndex('user', username, uid); }); } @@ -407,7 +406,7 @@ var bcrypt = require('bcrypt'), callback([]); return; } - userSearch.query(username).type('or').end(function(err, uids) { + db.search('user', username, (function(err, uids) { if (err) { console.log(err); return; From e9fbab0f2667c232ff9d040edd1bf1fca9c8bed5 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 20:11:05 -0500 Subject: [PATCH 65/83] need mongo 2.4+ for text search' --- src/database/mongo.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 699c46af6f..9005c86095 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -69,22 +69,21 @@ // module.searchIndex = function(key, content, id) { - + // REQUIRES MONGO 2.4 db.collection('search').update({key:key, content:content, _id:id}, {upsert:true, w: 1}, function(err, result) { }); } module.search = function(key, term, callback) { + // REQUIRES MONGO 2.4 db.command({text:"search" , search: term }, function(err, result) { callback(err, result) }); - /*db.runCommand("text", { search: term }, function(err, result) { - callback(err, result); - });*/ } module.searchRemove = function(key, id) { + // REQUIRES MONGO 2.4 db.collection('search').remove({_id:id}, function(err, result) { callback(err, result); }); From 81055523a0dd0db5341aad9da49d7ae887ee2ea0 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 20:24:25 -0500 Subject: [PATCH 66/83] fixed bracket --- src/user.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/user.js b/src/user.js index c4459be153..a7b8c95ca1 100644 --- a/src/user.js +++ b/src/user.js @@ -406,7 +406,7 @@ var bcrypt = require('bcrypt'), callback([]); return; } - db.search('user', username, (function(err, uids) { + db.search('user', username, function(err, uids) { if (err) { console.log(err); return; From 3a7fcc2d3d0cbb12593fcd21473d4bb45f095b69 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 21:07:35 -0500 Subject: [PATCH 67/83] search in mongo --- src/database/mongo.js | 49 ++++++++++++++++++++++++++++++++----------- src/database/redis.js | 2 +- src/install.js | 1 - src/routes/debug.js | 5 +++-- 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 9005c86095..0c538cd103 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -25,6 +25,13 @@ }); + // TODO : what is the db user name?? + /*if(nconf.get('mongo:password')) { + db.authenticate(dbUser, nconf.get('mongo:password'), function (err) { + }); + }*/ + + db.createCollection('objects', function(err, collection) { if(err) { winston.error("Error creating collection " + err.message); @@ -56,11 +63,7 @@ callback(err); }); - // look up how its done in mongo - /*if (nconf.get('mongo:password')) { - redisClient.auth(nconf.get('mongo:password')); - } - */ + } @@ -69,22 +72,44 @@ // module.searchIndex = function(key, content, id) { - // REQUIRES MONGO 2.4 - db.collection('search').update({key:key, content:content, _id:id}, {upsert:true, w: 1}, function(err, result) { + var data = { + id:id, + key:key, + content:content + }; + + db.collection('search').update({id:id, key:key}, {$set:data}, {upsert:true, w: 1}, function(err, result) { + if(err) { + winston.error('Error indexing ' + err.message); + } }); } module.search = function(key, term, callback) { - // REQUIRES MONGO 2.4 - db.command({text:"search" , search: term }, function(err, result) { - callback(err, result) + + db.command({text:"search" , search: term, filter: {key:key} }, function(err, result) { + if(err) { + return callback(err); + } + + if(!result) { + return callback(null, []); + } + + if(result.results && result.results.length) { + var data = result.results.map(function(item) { + return item.obj.id; + }); + callback(null, data); + } else { + callback(null, []); + } }); } module.searchRemove = function(key, id) { - // REQUIRES MONGO 2.4 - db.collection('search').remove({_id:id}, function(err, result) { + db.collection('search').remove({id:id, key:key}, function(err, result) { callback(err, result); }); } diff --git a/src/database/redis.js b/src/database/redis.js index 7361be3458..b3a27ef895 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -86,7 +86,7 @@ module.search = function(key, term, callback) { function search(searchObj, callback) { searchObj - .query(query = term).type('or') + .query(term).type('or') .end(callback); } diff --git a/src/install.js b/src/install.js index 22d00cfdd3..a695fe3b4d 100644 --- a/src/install.js +++ b/src/install.js @@ -219,7 +219,6 @@ var async = require('async'), async.each(defaults, function (configObj, next) { meta.configs.setOnEmpty(configObj.field, configObj.value, next); }, function (err) { - console.log('calling meta.configs.init'); meta.configs.init(next); }); }, diff --git a/src/routes/debug.js b/src/routes/debug.js index 59838d83b1..6b413ee007 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -407,10 +407,11 @@ var DebugRoute = function(app) { var miscTests = [ function(next) { - //db.searchIndex('post', 'here is content tomato testing purple orange', 1); + /*db.searchIndex('post', 'green tomato is healthy', 2); + next();*/ db.search('post', 'tomato', function(err, result) { next(err, result); - }) + }); } ]; From 097810a057f5f4830802ec40772ae59a10dd71c2 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 21:29:51 -0500 Subject: [PATCH 68/83] parseInt fixes getObjects fix --- src/database/mongo.js | 46 ++++++++++++++++++++++++------------------- src/groups.js | 2 +- src/messaging.js | 4 ++-- src/posts.js | 4 +++- src/routes/debug.js | 5 +++-- src/routes/user.js | 8 +++++--- src/sitemap.js | 2 +- src/user.js | 4 ++-- src/webserver.js | 2 +- 9 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 0c538cd103..0c7552ff2f 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -62,8 +62,6 @@ callback(err); }); - - } @@ -132,20 +130,15 @@ module.info = function(callback) { db.stats({scale:1024}, function(err, stats) { - db.serverStatus(function(err, serverStatus) { - - stats.avgObjSize = (stats.avgObjSize / 1024).toFixed(2); + stats.avgObjSize = (stats.avgObjSize / 1024).toFixed(2); - stats.serverStatus = serverStatus; + stats.raw = JSON.stringify(stats, null, 4); - stats.raw = JSON.stringify(stats, null, 4); + stats.mongo = true; + //remove this when andrew adds in undefined checking to templates + stats.redis = false; + callback(err, stats); - stats.mongo = true; - //remove this when andrew adds in undefined checking to templates - stats.redis = false; - callback(err, stats); - - }); }); } @@ -242,6 +235,7 @@ } module.getObjects = function(keys, callback) { + db.collection('objects').find({_key:{$in:keys}}, {_id:0}).toArray(function(err, data) { if(err) { @@ -251,15 +245,27 @@ var returnData = [], resultIndex = 0; - for(var i=0; i Date: Thu, 5 Dec 2013 21:45:21 -0500 Subject: [PATCH 69/83] remove dataFileVersion it breaks templates --- src/database/mongo.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/database/mongo.js b/src/database/mongo.js index 0c7552ff2f..149741d715 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -130,6 +130,10 @@ module.info = function(callback) { db.stats({scale:1024}, function(err, stats) { + // TODO : if this it not deleted the templates break, + // it is a nested object inside stats + delete stats.dataFileVersion; + stats.avgObjSize = (stats.avgObjSize / 1024).toFixed(2); stats.raw = JSON.stringify(stats, null, 4); From 5f86e31d1e0916835693b598b867c86cfb26b56d Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 21:51:05 -0500 Subject: [PATCH 70/83] fixed favouriting --- src/database/mongo.js | 4 +++- src/favourites.js | 6 ++++-- src/topics.js | 2 +- src/websockets.js | 1 + 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 149741d715..45df048646 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -108,7 +108,9 @@ module.searchRemove = function(key, id) { db.collection('search').remove({id:id, key:key}, function(err, result) { - callback(err, result); + if(err) { + winston.error('Error removing search ' + err.message); + } }); } diff --git a/src/favourites.js b/src/favourites.js index 36214e0d25..acf6e35d60 100644 --- a/src/favourites.js +++ b/src/favourites.js @@ -8,6 +8,7 @@ var db = require('./database'), "use strict"; Favourites.favourite = function (pid, room_id, uid, socket) { + if (uid === 0) { translator.mget(['topic:favourites.not_logged_in.message', 'topic:favourites.not_logged_in.title'], function(err, results) { @@ -25,7 +26,8 @@ var db = require('./database'), posts.getPostFields(pid, ['uid', 'timestamp'], function (err, postData) { Favourites.hasFavourited(pid, uid, function (hasFavourited) { - if (hasFavourited === 0) { + + if (!hasFavourited) { db.setAdd('pid:' + pid + ':users_favourited', uid); db.sortedSetAdd('uid:' + uid + ':favourites', postData.timestamp, pid); @@ -60,7 +62,7 @@ var db = require('./database'), posts.getPostField(pid, 'uid', function (err, uid_of_poster) { Favourites.hasFavourited(pid, uid, function (hasFavourited) { - if (hasFavourited === 1) { + if (hasFavourited) { db.setRemove('pid:' + pid + ':users_favourited', uid); db.sortedSetRemove('uid:' + uid + ':favourites', pid); diff --git a/src/topics.js b/src/topics.js index 9e4e4b2eb3..1865017471 100644 --- a/src/topics.js +++ b/src/topics.js @@ -207,7 +207,7 @@ var async = require('async'), privileges = results[2]; for (var i = 0; i < postData.length; ++i) { - postData[i].favourited = fav_data[postData[i].pid] === 1; + postData[i].favourited = fav_data[postData[i].pid]; postData[i].display_moderator_tools = ((current_user != 0) && (postData[i].uid == current_user || privileges.editable)); } diff --git a/src/websockets.js b/src/websockets.js index 8cd9672538..1850c05d0b 100644 --- a/src/websockets.js +++ b/src/websockets.js @@ -504,6 +504,7 @@ websockets.init = function(io) { }); socket.on('api:posts.favourite', function(data) { + console.log('fave'); favourites.favourite(data.pid, data.room_id, uid, socket); }); From 1c23be891101209482af179828e28b85f7529e1f Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Thu, 5 Dec 2013 21:53:24 -0500 Subject: [PATCH 71/83] removed console.log --- src/websockets.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/websockets.js b/src/websockets.js index 1850c05d0b..8cd9672538 100644 --- a/src/websockets.js +++ b/src/websockets.js @@ -504,7 +504,6 @@ websockets.init = function(io) { }); socket.on('api:posts.favourite', function(data) { - console.log('fave'); favourites.favourite(data.pid, data.room_id, uid, socket); }); From 0da141e7bcb2eee49927060ec28cccf207ec70b5 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Fri, 6 Dec 2013 13:21:21 -0500 Subject: [PATCH 72/83] removed redismock, added database mocked, fixed tests to work with dbal --- mocks/databasemock.js | 81 +++++++++++++++++++++++++++++++++ mocks/redismock.js | 69 ---------------------------- src/database/mongo.js | 37 +++++++++++---- src/websockets.js | 6 ++- tests/categories.js | 18 ++------ tests/{redis.js => database.js} | 2 +- tests/topics.js | 17 ++----- tests/user.js | 10 +--- 8 files changed, 124 insertions(+), 116 deletions(-) create mode 100644 mocks/databasemock.js delete mode 100644 mocks/redismock.js rename tests/{redis.js => database.js} (76%) diff --git a/mocks/databasemock.js b/mocks/databasemock.js new file mode 100644 index 0000000000..89e5f850e7 --- /dev/null +++ b/mocks/databasemock.js @@ -0,0 +1,81 @@ +/** + * Database Mock - wrapper for database.js, makes system use separate test db, instead of production + * ATTENTION: testing db is flushed before every use! + */ + +(function(module) { + 'use strict'; + + var utils = require('./../public/src/utils.js'), + path = require('path'), + nconf = require('nconf'), + winston = require('winston'), + errorText; + + + nconf.file({ file: path.join(__dirname, '../config.json') }); + + var dbType = nconf.get('database'), + testDbConfig = nconf.get('test_database'), + productionDbConfig = nconf.get(dbType); + + if(!testDbConfig){ + errorText = 'test_database is not defined'; + winston.info( + "\n===========================================================\n"+ + "Please, add parameters for test database in config.json\n"+ + "For example (redis):\n"+ + '"test_database": {' + '\n' + + ' "host": "127.0.0.1",' + '\n' + + ' "port": "6379",' + '\n' + + ' "password": "",' + '\n' + + ' "database": "1"' + '\n' + + '}\n'+ + " or (mongo):\n" + + '"test_database": {' + '\n' + + ' "host": "127.0.0.1",' + '\n' + + ' "port": "27017",' + '\n' + + ' "password": "",' + '\n' + + ' "database": "1"' + '\n' + + '}\n'+ + "===========================================================" + ); + winston.error(errorText); + throw new Error(errorText); + } + + if( testDbConfig.database === productionDbConfig.database && + testDbConfig.host === productionDbConfig.host && + testDbConfig.port === productionDbConfig.port + ){ + errorText = 'test_database has the same config as production db'; + winston.error(errorText); + throw new Error(errorText); + } + + nconf.set(dbType, testDbConfig); + + db = require('../src/database'); + before(function(done) { + + db.init(function(err) { + //Clean up + db.flushdb(function(err) { + if(err){ + winston.error(err); + throw new Error(err); + } else { + winston.info('test_database flushed'); + done(); + } + + //TODO: data seeding, if needed at all + + + }); + }); + }); + + module.exports = db; + +}(module)); diff --git a/mocks/redismock.js b/mocks/redismock.js deleted file mode 100644 index cfdbb4d9fe..0000000000 --- a/mocks/redismock.js +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Redis Mock - wrapper for redis.js, makes system use separate test db, instead of production - * ATTENTION: testing db is flushed before every use! - */ - -(function(module) { - 'use strict'; - - var RedisDB, - redis = require('redis'), - utils = require('./../public/src/utils.js'), - path = require('path'), - nconf = require('nconf'), - winston = require('winston'), - errorText; - - - nconf.file({ file: path.join(__dirname, '../config.json') }); - - var testDbConfig = nconf.get('redis_test'), - productionDbConfig = nconf.get('redis'); - if(!testDbConfig){ - errorText = 'redis_test database is not defined'; - winston.info( - "\n===========================================================\n"+ - "Please, add parameters for test database in config.json\n"+ - "For example:\n"+ - '"redis_test": {' + '\n' + - ' "host": "127.0.0.1",' + '\n' + - ' "port": "6379",' + '\n' + - ' "password": "",' + '\n' + - ' "database": "1"' + '\n' + - '}\n'+ - "===========================================================" - ); - winston.error(errorText); - throw new Error(errorText); - } - - if( testDbConfig.database === productionDbConfig.database && - testDbConfig.host === productionDbConfig.host && - testDbConfig.port === productionDbConfig.port - ){ - errorText = 'redis_test database has the same config as production db'; - winston.error(errorText); - throw new Error(errorText); - } - - nconf.set('redis',testDbConfig); - - RedisDB = require('../src/redis.js'); - - - //Clean up - RedisDB.send_command('flushdb', [], function(error){ - if(error){ - winston.error(error); - throw new Error(error); - } else { - winston.info('redis_test db flushed'); - } - }); - - //TODO: data seeding, if needed at all - - - module.exports = RedisDB; - -}(module)); diff --git a/src/database/mongo.js b/src/database/mongo.js index 45df048646..0d5387ff93 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -118,9 +118,14 @@ db.dropDatabase(function(err, result) { if(err){ winston.error(error); - return callback(err); + if(callback) { + return callback(err); + } + } + + if(callback) { + callback(null); } - callback(null); }); } @@ -379,7 +384,10 @@ // sets module.setAdd = function(key, value, callback) { - db.collection('objects').update({_key:key}, {$addToSet: { members: value.toString() }}, {upsert:true, w: 1}, function(err, result) { + if(value !== null && value !== undefined) { + value = value.toString(); + } + db.collection('objects').update({_key:key}, {$addToSet: { members: value }}, {upsert:true, w: 1}, function(err, result) { if(callback) { callback(err, result); } @@ -387,7 +395,10 @@ } module.setRemove = function(key, value, callback) { - db.collection('objects').update({_key:key, members: value.toString()}, {$pull : {members: value}}, function(err, result) { + if(value !== null && value !== undefined) { + value = value.toString(); + } + db.collection('objects').update({_key:key, members: value}, {$pull : {members: value}}, function(err, result) { if(callback) { callback(err, result); } @@ -395,7 +406,10 @@ } module.isSetMember = function(key, value, callback) { - db.collection('objects').findOne({_key:key, members: value.toString()}, function(err, item) { + if(value !== null && value !== undefined) { + value = value.toString(); + } + db.collection('objects').findOne({_key:key, members: value}, function(err, item) { callback(err, item !== null && item !== undefined); }); } @@ -650,13 +664,20 @@ db.collection('objects').update({_key: key }, { $pop: { array: 1 } }, function(err, result) { if(err) { - return callback(err); + if(callback) { + return callback(err); + } + return; } if(value && value.length) { - callback(err, value[0]); + if(callback) { + callback(err, value[0]); + } } else { - callback(err, null); + if(callback) { + callback(err, null); + } } }); }); diff --git a/src/websockets.js b/src/websockets.js index 992a8fcba0..8a76ca3402 100644 --- a/src/websockets.js +++ b/src/websockets.js @@ -1152,8 +1152,10 @@ websockets.init = function(io) { return io.sockets.in(room); }; +} + websockets.getConnectedClients = function() { return userSockets; } -} -})(module.exports); + +})(module.exports); diff --git a/tests/categories.js b/tests/categories.js index 36a4d63925..070a71e884 100644 --- a/tests/categories.js +++ b/tests/categories.js @@ -7,14 +7,7 @@ process.on('uncaughtException', function (err) { }); var assert = require('assert'), - RDB = require('../mocks/redismock'); - -// Reds is not technically used in this test suite, but its invocation is required to stop the included -// libraries from trying to connect to the default Redis host/port -var reds = require('reds'); -reds.createClient = function () { - return reds.client || (reds.client = RDB); -}; + db = require('../mocks/databasemock'); var Categories = require('../src/categories'); @@ -23,6 +16,7 @@ describe('Categories', function() { describe('.create', function() { it('should create a new category', function(done) { + Categories.create({ name: 'Test Category', description: 'Test category created by testing script', @@ -62,11 +56,7 @@ describe('Categories', function() { }); after(function() { - // TODO : replace with dbal - RDB.multi() - .del('category:'+categoryObj.cid) - .rpop('categories:cid') - .exec(); - + db.delete('category:' + categoryObj.cid); + db.listRemoveLast('categories:cid'); }); }); \ No newline at end of file diff --git a/tests/redis.js b/tests/database.js similarity index 76% rename from tests/redis.js rename to tests/database.js index 0af4f1a120..c857233c53 100644 --- a/tests/redis.js +++ b/tests/database.js @@ -4,7 +4,7 @@ var assert = require('assert'); describe('Test database', function() { it('should work', function(){ assert.doesNotThrow(function(){ - var RDB = require('../mocks/redismock'); + var db = require('../mocks/databasemock'); }); }); }); diff --git a/tests/topics.js b/tests/topics.js index 063451cc3e..9d27db8cf7 100644 --- a/tests/topics.js +++ b/tests/topics.js @@ -5,14 +5,8 @@ process.on('uncaughtException', function (err) { }); var assert = require('assert'), - RDB = require('../mocks/redismock'); + db = require('../mocks/databasemock'); -// Reds is not technically used in this test suite, but its invocation is required to stop the included -// libraries from trying to connect to the default Redis host/port -var reds = require('reds'); -reds.createClient = function () { - return reds.client || (reds.client = RDB); -}; var Topics = require('../src/topics'); @@ -43,7 +37,7 @@ describe('Topic\'s', function() { topic.userId = null; Topics.post(topic.userId, topic.title, topic.content, topic.categoryId, function(err, result) { - assert.equal(err.message, 'not-logged-in'); + assert.equal(err.message, 'invalid-user'); done(); }); }); @@ -75,11 +69,6 @@ describe('Topic\'s', function() { }); after(function() { - RDB.send_command('flushdb', [], function(error){ - if(error){ - winston.error(error); - throw new Error(error); - } - }); + db.flushdb(); }); }); \ No newline at end of file diff --git a/tests/user.js b/tests/user.js index cb89ac1fc9..7adfca61ee 100644 --- a/tests/user.js +++ b/tests/user.js @@ -7,7 +7,7 @@ process.on('uncaughtException', function (err) { }); var assert = require('assert'), - RDB = require('../mocks/redismock'); + db = require('../mocks/databasemock'); var User = require('../src/user'); @@ -43,12 +43,6 @@ describe('User', function() { }); after(function() { - //Clean up - RDB.send_command('flushdb', [], function(error){ - if(error){ - winston.error(error); - throw new Error(error); - } - }); + db.flushdb(); }); }); \ No newline at end of file From df10bde2dbf75f58f38ed53f54af80c18fd515b3 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Fri, 6 Dec 2013 13:30:44 -0500 Subject: [PATCH 73/83] added 1 more parseInt --- src/topics.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topics.js b/src/topics.js index 1865017471..f5a496bc77 100644 --- a/src/topics.js +++ b/src/topics.js @@ -574,7 +574,7 @@ var async = require('async'), 'slug': topicData.slug, 'postcount': topicData.postcount, 'viewcount': topicData.viewcount, - 'unreplied': topicData.postcount > 1, + 'unreplied': parseInt(topicData.postcount, 10) > 1, 'topic_id': tid, 'expose_tools': privileges.editable ? 1 : 0, 'posts': topicPosts From 2b7e4cbdf496dedb5e3223fb7fdf42fd240afdbb Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Fri, 6 Dec 2013 13:46:12 -0500 Subject: [PATCH 74/83] moved tests from debug to tests folder --- src/routes/debug.js | 354 -------------------------------------------- tests/database.js | 322 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 321 insertions(+), 355 deletions(-) diff --git a/src/routes/debug.js b/src/routes/debug.js index ab3258def3..d089a48b2c 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -78,360 +78,6 @@ var DebugRoute = function(app) { }); }); }); - - app.get('/prune', function(req, res) { - require('./../notifications').prune(); - - //function(err, result) { - // if(err) { - // res.send(err.message); - // return; - // } - res.send('done'); - //} - }); - - app.get('/mongo', function(req, res) { - var dbtype = 'mongo'; - if(req.query.db) { - dbtype = req.query.db; - } - - var db = require('./../database/' + dbtype); - - - var objectKey = 'testing4'; - - function createUser(callback) { - user.create('baris','123456', 'barisusakli@gmail.com', callback); - } - - function getUser(callback) { - user.getUserData(1, callback); - } - - function setObject(callback) { - db.setObject(objectKey, {name:'baris', 'lastname':'usakli', age:3}, function(err, result) { - console.log('setObject return ', result); - callback(err, {'setObject':result}); - }); - } - - function getObject(callback) { - db.getObject(objectKey, function(err, data) { - console.log('getObject return ', data); - callback(err, {'getObject':data}); - }); - } - - function getObjects(callback) { - db.getObjects(['testing1', 'testing2', 'retardation', 'user:1'], function(err, data) { - console.log('getObjects return ', data); - callback(err, {'getObjects':data}); - }); - } - - function setObjectField(callback) { - db.setObjectField(objectKey, 'reputation', 5, function(err, result) { - console.log('setObjectField return', result); - callback(err, {'setObjectField': result}); - }); - } - - function getObjectField(callback) { - db.getObjectField(objectKey, 'age', function(err, age) { - console.log('getObjectField return', age); - callback(err, {'getObjectField' : age}); - }); - } - - function getObjectFields(callback) { - db.getObjectFields(objectKey, ['name', 'lastname'], function(err, data) { - console.log('getObjectFields return', data); - callback(err, {'getObjectFields':data}); - }); - } - - function getObjectValues(callback) { - db.getObjectValues(objectKey, function(err, data) { - console.log('getObjectValues return', data); - callback(err, {'getObjectValues':data}); - }); - } - - function isObjectField(callback) { - db.isObjectField(objectKey, 'age', function(err, data) { - console.log('isObjectField return', data); - callback(err, {'isObjectField':data}); - }); - } - - function deleteObjectField(callback) { - db.deleteObjectField(objectKey, 'reputation', function(err, data) { - console.log('deleteObjectField return', data); - callback(err, {'deleteObjectField':data}); - }); - } - - function incrObjectFieldBy(callback) { - db.incrObjectFieldBy(objectKey, 'age', 3, function(err, data) { - console.log('incrObjectFieldBy return', data); - callback(err, {'incrObjectFieldBy':data}); - }); - } - - - function sortedSetAdd(callback) { - db.sortedSetAdd('sortedSet3', 12, 5, function(err, data) { - console.log('sortedSetAdd return', data); - callback(err, {'sortedSetAdd': data}); - }); - } - - function sortedSetRemove(callback) { - db.sortedSetRemove('sortedSet3', 12, function(err, data) { - console.log('sortedSetRemove return', data); - callback(err, {'sortedSetRemove': data}); - }); - } - - function getSortedSetRange(callback) { - db.getSortedSetRevRange('sortedSet3', 0, -1, function(err, data) { - console.log('getSortedSetRange return', data); - callback(err, {'getSortedSetRange': data}); - }); - /*var args = ['sortedSet2', '+inf', 100, 'LIMIT', 0, 10]; - db.getSortedSetRevRangeByScore(args, function(err, data) { - console.log('getSortedSetRevRangeByScore return', data); - callback(err, {'getSortedSetRevRangeByScore': data}); - });*/ - } - - function sortedSetCount(callback) { - db.sortedSetCount('sortedSet3', -Infinity, Infinity, function(err, data) { - console.log('sortedSetCount return', data); - callback(err, {'sortedSetCount': data}); - }); - } - - function sortedSetScore(callback) { - db.sortedSetScore('users:joindate', 1, function(err, data) { - console.log('sortedSetScore return', data); - callback(err, {'sortedSetScore': data}); - }); - } - - function sortedSetsScore(callback) { - db.sortedSetsScore(['users:joindate', 'users:derp', 'users:postcount'], 1, function(err, data) { - console.log('sortedSetsScore return', data); - callback(err, {'sortedSetsScore': data}); - }); - } - - function listAppend(callback) { - db.listAppend('myList5', 5, function(err, data) { - console.log('listAppend return', data); - callback(err, {'listAppend': data}); - }); - } - - function listPrepend(callback) { - db.listPrepend('myList5', 4, function(err, data) { - console.log('listPrepend return', data); - callback(err, {'listPrepend': data}); - }); - } - - - function listRemoveLast(callback) { - db.listRemoveLast('myList5', function(err, data) { - console.log('listRemoveLast return', data); - callback(err, {'listRemoveLast': data}); - }); - } - - - function getListRange(callback) { - db.getListRange('myList5', 0, -1, function(err, data) { - console.log('getListRange return', data); - callback(err, {'getListRange': data}); - }); - } - - function get(callback) { - db.get('testingStr', function(err, data) { - console.log('get return', data); - callback(err, {'get': data}); - }); - } - - function set(callback) { - db.set('testingStr', 'opppa gangastayla', function(err, data) { - console.log('set return', data); - callback(err, {'set': data}); - }); - } - - function deleteKey(callback) { - db.delete('testingStr', function(err, data) { - console.log('delete return', data); - callback(err, {'delete': data}); - }); - } - - function exists(callback) { - db.exists('testingStr', function(err, data) { - console.log('exists return', data); - callback(err, {'exists': data}); - }); - } - - - function setAdd(callback) { - db.setAdd('myTestSet', 15, function(err, data) { - console.log('setAdd return', data); - callback(err, {'setAdd': data}); - }); - } - - function setRemove(callback) { - db.setRemove('myTestSet', 15, function(err, data) { - console.log('setRemove return', data); - callback(err, {'setRemove': data}); - }); - } - - function getSetMembers(callback) { - db.getSetMembers('myTestSet', function(err, data) { - console.log('getSetMembers return', data); - callback(err, {'getSetMembers': data}); - }); - } - - function isSetMember(callback) { - db.isSetMember('myTestSet', 15, function(err, data) { - console.log('isSetMember return', data); - callback(err, {'isSetMember': data}); - }); - } - - function isMemberOfSets(callback) { - db.isMemberOfSets(['doesntexist', 'myTestSet', 'nonexistingSet'], 15, function(err, data) { - console.log('isMemberOfSets return', data); - callback(err, {'isMemberOfSets': data}); - }); - } - - function setRemoveRandom(callback) { - db.setRemoveRandom('myTestSet', function(err, data) { - console.log('setRemoveRandom return', data); - callback(err, {'setRemoveRandom': data}); - }); - } - - function setCount(callback) { - db.setCount('myTestSet', function(err, data) { - console.log('setCount return', data); - callback(err, {'setCount': data}); - }); - } - - - - var objectTasks = [ - //createUser, - getUser, - setObject, - getObject, - deleteObjectField, - getObject, - setObjectField, - getObject, - deleteObjectField, - getObject, - getObjectField, - getObjectFields, - getObjectValues, - isObjectField, - incrObjectFieldBy, - getObject, - getObjects - ]; - - var sortedSetTasks = [ - //sortedSetAdd, - getSortedSetRange, - sortedSetAdd, - getSortedSetRange, - //sortedSetRemove, - getSortedSetRange, - sortedSetCount, - sortedSetScore, - sortedSetsScore - ]; - - var listTasks = [ - //listAppend, - //listPrepend, - getListRange, - listRemoveLast, - getListRange - ]; - - var keyTasks = [ - get, - set, - get, - exists, - deleteKey, - deleteKey, - get, - exists - ]; - - var setTasks = [ - getSetMembers, - setAdd, - getSetMembers, - setRemove, - getSetMembers, - isSetMember, - setAdd, - getSetMembers, - isSetMember, - setRemoveRandom, - getSetMembers, - setCount, - isMemberOfSets - ]; - - var miscTests = [ - function(next) { - /*db.searchIndex('post', 'green tomato is healthy', 2); - next();*/ - /*db.search('post', 'tomato', function(err, result) { - next(err, result); - });*/ - db.getObjects(['deeerp', 'user:1', 'bonanza', 'post:1'], next); - - } - ]; - - require('async').series(miscTests, function(err, results) { - if(err) { - console.log(err); - res.send(err.message); - } else { - res.json(results); - } - }); - }); - - app.get('/test', function(req, res) { - topics.pushUnreadCount(); - res.send(); - }); - }); }; diff --git a/tests/database.js b/tests/database.js index c857233c53..071b291957 100644 --- a/tests/database.js +++ b/tests/database.js @@ -1,4 +1,6 @@ -var assert = require('assert'); +var assert = require('assert'), + db = require('../mocks/databasemock'), + async = require('async'); describe('Test database', function() { @@ -7,4 +9,322 @@ describe('Test database', function() { var db = require('../mocks/databasemock'); }); }); + + it('should not throw err', function(done) { + var objectKey = 'testObj'; + + function setObject(callback) { + db.setObject(objectKey, {name:'baris', 'lastname':'usakli', age:3}, function(err, result) { + callback(err, {'setObject':result}); + }); + } + + function getObject(callback) { + db.getObject(objectKey, function(err, data) { + callback(err, {'getObject':data}); + }); + } + + function getObjects(callback) { + db.getObjects(['testing1', objectKey, 'doesntexist', 'user:1'], function(err, data) { + callback(err, {'getObjects':data}); + }); + } + + function setObjectField(callback) { + db.setObjectField(objectKey, 'reputation', 5, function(err, result) { + callback(err, {'setObjectField': result}); + }); + } + + function getObjectField(callback) { + db.getObjectField(objectKey, 'age', function(err, age) { + callback(err, {'getObjectField' : age}); + }); + } + + function getObjectFields(callback) { + db.getObjectFields(objectKey, ['name', 'lastname'], function(err, data) { + callback(err, {'getObjectFields':data}); + }); + } + + function getObjectValues(callback) { + db.getObjectValues(objectKey, function(err, data) { + callback(err, {'getObjectValues':data}); + }); + } + + function isObjectField(callback) { + db.isObjectField(objectKey, 'age', function(err, data) { + callback(err, {'isObjectField':data}); + }); + } + + function deleteObjectField(callback) { + db.deleteObjectField(objectKey, 'reputation', function(err, data) { + callback(err, {'deleteObjectField':data}); + }); + } + + function incrObjectFieldBy(callback) { + db.incrObjectFieldBy(objectKey, 'age', 3, function(err, data) { + callback(err, {'incrObjectFieldBy':data}); + }); + } + + var objectTasks = [ + setObject, + getObject, + deleteObjectField, + getObject, + setObjectField, + getObject, + deleteObjectField, + getObject, + getObjectField, + getObjectFields, + getObjectValues, + isObjectField, + incrObjectFieldBy, + getObject, + getObjects + ]; + + async.series(objectTasks, function(err, results) { + assert.equal(err, null, 'error in object methods'); + assert.ok(results); + + done(); + }); + }); + + it('should not throw err', function(done) { + + function sortedSetAdd(callback) { + db.sortedSetAdd('sortedSet3', 12, 5, function(err, data) { + callback(err, {'sortedSetAdd': data}); + }); + } + + function sortedSetRemove(callback) { + db.sortedSetRemove('sortedSet3', 12, function(err, data) { + callback(err, {'sortedSetRemove': data}); + }); + } + + function getSortedSetRange(callback) { + db.getSortedSetRevRange('sortedSet3', 0, -1, function(err, data) { + callback(err, {'getSortedSetRange': data}); + }); + } + + function getSortedSetRevRangeByScore(callback) { + var args = ['sortedSet2', '+inf', 100, 'LIMIT', 0, 10]; + db.getSortedSetRevRangeByScore(args, function(err, data) { + callback(err, {'getSortedSetRevRangeByScore': data}); + }); + } + + function sortedSetCount(callback) { + db.sortedSetCount('sortedSet3', -Infinity, Infinity, function(err, data) { + callback(err, {'sortedSetCount': data}); + }); + } + + function sortedSetScore(callback) { + db.sortedSetScore('users:joindate', 1, function(err, data) { + callback(err, {'sortedSetScore': data}); + }); + } + + function sortedSetsScore(callback) { + db.sortedSetsScore(['users:joindate', 'users:derp', 'users:postcount'], 1, function(err, data) { + callback(err, {'sortedSetsScore': data}); + }); + } + + var sortedSetTasks = [ + sortedSetAdd, + getSortedSetRange, + sortedSetAdd, + getSortedSetRange, + sortedSetRemove, + getSortedSetRange, + sortedSetCount, + sortedSetScore, + sortedSetsScore, + getSortedSetRevRangeByScore + ]; + + async.series(sortedSetTasks, function(err, results) { + assert.equal(err, null, 'error in sorted set methods'); + assert.ok(results); + + done(); + }); + + }); + + it('should not throw err', function(done) { + + function listAppend(callback) { + db.listAppend('myList5', 5, function(err, data) { + callback(err, {'listAppend': data}); + }); + } + + function listPrepend(callback) { + db.listPrepend('myList5', 4, function(err, data) { + callback(err, {'listPrepend': data}); + }); + } + + + function listRemoveLast(callback) { + db.listRemoveLast('myList5', function(err, data) { + callback(err, {'listRemoveLast': data}); + }); + } + + + function getListRange(callback) { + db.getListRange('myList5', 0, -1, function(err, data) { + callback(err, {'getListRange': data}); + }); + } + + var listTasks = [ + listAppend, + listPrepend, + getListRange, + listRemoveLast, + getListRange + ]; + + async.series(listTasks, function(err, results) { + assert.equal(err, null, 'error in list methods'); + assert.ok(results); + + done(); + }); + + }); + + it('should not throw err', function(done) { + + function get(callback) { + db.get('testingStr', function(err, data) { + callback(err, {'get': data}); + }); + } + + function set(callback) { + db.set('testingStr', 'opppa gangastayla', function(err, data) { + callback(err, {'set': data}); + }); + } + + function deleteKey(callback) { + db.delete('testingStr', function(err, data) { + callback(err, {'delete': data}); + }); + } + + function exists(callback) { + db.exists('testingStr', function(err, data) { + callback(err, {'exists': data}); + }); + } + + var keyTasks = [ + get, + set, + get, + exists, + deleteKey, + deleteKey, + get, + exists + ]; + + async.series(keyTasks, function(err, results) { + assert.equal(err, null, 'error in key methods'); + assert.ok(results); + + done(); + }); + + }); + + it('should not throw err', function(done) { + + + function setAdd(callback) { + db.setAdd('myTestSet', 15, function(err, data) { + callback(err, {'setAdd': data}); + }); + } + + function setRemove(callback) { + db.setRemove('myTestSet', 15, function(err, data) { + callback(err, {'setRemove': data}); + }); + } + + function getSetMembers(callback) { + db.getSetMembers('myTestSet', function(err, data) { + callback(err, {'getSetMembers': data}); + }); + } + + function isSetMember(callback) { + db.isSetMember('myTestSet', 15, function(err, data) { + callback(err, {'isSetMember': data}); + }); + } + + function isMemberOfSets(callback) { + db.isMemberOfSets(['doesntexist', 'myTestSet', 'nonexistingSet'], 15, function(err, data) { + callback(err, {'isMemberOfSets': data}); + }); + } + + function setRemoveRandom(callback) { + db.setRemoveRandom('myTestSet', function(err, data) { + callback(err, {'setRemoveRandom': data}); + }); + } + + function setCount(callback) { + db.setCount('myTestSet', function(err, data) { + callback(err, {'setCount': data}); + }); + } + + + var setTasks = [ + getSetMembers, + setAdd, + getSetMembers, + setRemove, + getSetMembers, + isSetMember, + setAdd, + getSetMembers, + isSetMember, + setRemoveRandom, + getSetMembers, + setCount, + isMemberOfSets + ]; + + + require('async').series(setTasks, function(err, results) { + assert.equal(err, null, 'error in set methods'); + assert.ok(results); + + done(); + }); + }); }); From f861d44d55738f9f85f984c09a12b2fab5626806 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Fri, 6 Dec 2013 14:22:31 -0500 Subject: [PATCH 75/83] updated upgrade script to use mongo or redis --- app.js | 8 +++--- src/database.js | 11 ++++++-- src/database/mongo.js | 2 ++ src/install.js | 4 +-- src/upgrade.js | 63 ++++++++++++++++++++++++++++++++++++++----- 5 files changed, 74 insertions(+), 14 deletions(-) diff --git a/app.js b/app.js index 103a2d19e1..8defae9ab2 100644 --- a/app.js +++ b/app.js @@ -154,10 +154,12 @@ nconf.file({ file: __dirname + '/config.json' }); - meta = require('./src/meta.js'); + require('./src/database').init(function(err) { + meta = require('./src/meta.js'); - meta.configs.init(function () { - require('./src/upgrade').upgrade(); + meta.configs.init(function () { + require('./src/upgrade').upgrade(); + }); }); } else/* if (nconf.get('help') */{ winston.info('Usage: node app [options] [arguments]'); diff --git a/src/database.js b/src/database.js index 465cd3bcd5..47c76c30c1 100644 --- a/src/database.js +++ b/src/database.js @@ -1,6 +1,13 @@ -var nconf = require('nconf'); - db = require('./database/' + nconf.get('database')); +var nconf = require('nconf'), + databaseType = nconf.get('database'); + + if(!databaseType) { + winston.info('Database type not set! Run npm app --setup'); + process.exit(); + } + +var db = require('./database/' + databaseType); module.exports = db; \ No newline at end of file diff --git a/src/database/mongo.js b/src/database/mongo.js index 0d5387ff93..b31d99b558 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -20,6 +20,8 @@ process.exit(); } + module.client = db; + module.sessionStore = new mongoStore({ db: db }); diff --git a/src/install.js b/src/install.js index a695fe3b4d..f7583c81fb 100644 --- a/src/install.js +++ b/src/install.js @@ -295,9 +295,7 @@ var async = require('async'), }, next); }, function (next) { - // Upgrading schema - var Upgrade = require('./upgrade'); - Upgrade.upgrade(next); + require('./upgrade').upgrade(next); } ], function (err) { if (err) { diff --git a/src/upgrade.js b/src/upgrade.js index 49bedc2a2c..99657007f7 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -1,14 +1,11 @@ "use strict"; -var //db = require('./database'), - - // TODO: temp until upgrade is figured out with dbal, - RDB = require('./database/redis').client, - +var db = require('./database'), async = require('async'), winston = require('winston'), notifications = require('./notifications'), categories = require('./categories'), + nconf = require('nconf'), Upgrade = {}, schemaDate, thisSchemaDate; @@ -17,7 +14,7 @@ Upgrade.check = function(callback) { // IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema var latestSchema = new Date(2013, 11, 2).getTime(); - RDB.get('schemaDate', function(err, value) { + db.get('schemaDate', function(err, value) { if (parseInt(value, 10) >= latestSchema) { callback(true); } else { @@ -27,6 +24,22 @@ Upgrade.check = function(callback) { }; Upgrade.upgrade = function(callback) { + var databaseType = nconf.get('database'); + + if(databaseType === 'redis') { + Upgrade.upgradeRedis(callback); + } else if(databaseType === 'mongo') { + Upgrade.upgradeMongo(callback); + } else { + winston.error('Unknown database type. Aborting upgrade'); + callback(new Error('unknown-database')); + } +}; + +Upgrade.upgradeRedis = function(callback) { + + var RDB = db.client; + winston.info('Beginning Redis database schema update'); async.series([ @@ -288,4 +301,42 @@ Upgrade.upgrade = function(callback) { }); }; +Upgrade.upgradeMongo = function(callback) { + var MDB = db.client; + + winston.info('Beginning Mongo database schema update'); + + async.series([ + function(next) { + db.get('schemaDate', function(err, value) { + console.log(schemaDate) + schemaDate = value; + thisSchemaDate = new Date(2013, 11, 6).getTime(); + next(); + }); + } + // Add new schema updates here + + ], function(err) { + if (!err) { + db.set('schemaDate', thisSchemaDate, function(err) { + if (!err) { + winston.info('[upgrade] Mongo schema update complete!'); + if (callback) { + callback(err); + } else { + process.exit(); + } + } else { + winston.error('[upgrade] Could not update NodeBB schema date!'); + process.exit(); + } + }); + } else { + winston.error('[upgrade] Errors were encountered while updating the NodeBB schema: ' + err.message); + process.exit(); + } + }); +} + module.exports = Upgrade; \ No newline at end of file From 351b07bb3470340e090a36f5435a65d5e532db26 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Fri, 6 Dec 2013 14:31:11 -0500 Subject: [PATCH 76/83] added authentication to mongo --- src/database/mongo.js | 12 ++++++++---- src/install.js | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index b31d99b558..211040ad2d 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -28,10 +28,14 @@ // TODO : what is the db user name?? - /*if(nconf.get('mongo:password')) { - db.authenticate(dbUser, nconf.get('mongo:password'), function (err) { + if(nconf.get('mongo:password') && nconf.get('mongo:username')) { + db.authenticate(nconf.get('mongo:username'), nconf.get('mongo:password'), function (err) { + if(err) { + winston.error(err.message); + } + process.exit(); }); - }*/ + } db.createCollection('objects', function(err, collection) { @@ -60,7 +64,7 @@ } }); } - }) + }); callback(err); }); diff --git a/src/install.js b/src/install.js index f7583c81fb..4deca311cd 100644 --- a/src/install.js +++ b/src/install.js @@ -64,6 +64,9 @@ var async = require('async'), name: 'mongo:port', description: 'Host port of your MongoDB instance', 'default': nconf.get('mongo:port') || 27017 + }, { + name: 'mongo:user', + description: 'MongoDB username' }, { name: 'mongo:password', description: 'Password of your MongoDB database' From 400845ce6c389e5e36d8da3b58d9e6c2327cce5b Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Fri, 6 Dec 2013 14:34:25 -0500 Subject: [PATCH 77/83] cleanup mongo init --- src/database/mongo.js | 67 ++++++++++++++++++++++--------------------- src/upgrade.js | 1 - 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index 211040ad2d..e9cfaa68e5 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -13,13 +13,13 @@ module.init = function(callback) { mongoClient.connect('mongodb://'+ mongoHost + ':' + nconf.get('mongo:port') + '/' + nconf.get('mongo:database'), function(err, _db) { - db = _db; - if(err) { winston.error("NodeBB could not connect to your Mongo database. Mongo returned the following error: " + err.message); process.exit(); } + db = _db; + module.client = db; module.sessionStore = new mongoStore({ @@ -27,46 +27,49 @@ }); - // TODO : what is the db user name?? if(nconf.get('mongo:password') && nconf.get('mongo:username')) { db.authenticate(nconf.get('mongo:username'), nconf.get('mongo:password'), function (err) { if(err) { winston.error(err.message); + process.exit(); } - process.exit(); + createCollections(); }); + } else { + createCollections(); } + function createCollections() { + db.createCollection('objects', function(err, collection) { + if(err) { + winston.error("Error creating collection " + err.message); + return; + } + if(collection) { + collection.ensureIndex({_key :1, setName:1}, {background:true}, function(err, name){ + if(err) { + winston.error("Error creating index " + err.message); + } + }); + } + }); - db.createCollection('objects', function(err, collection) { - if(err) { - winston.error("Error creating collection " + err.message); - return; - } - if(collection) { - collection.ensureIndex({_key :1, setName:1}, {background:true}, function(err, name){ - if(err) { - winston.error("Error creating index " + err.message); - } - }); - } - }); - - db.createCollection('search', function(err, collection) { - if(err) { - winston.error("Error creating collection " + err.message); - return; - } - if(collection) { - collection.ensureIndex({content:'text'}, {background:true}, function(err, name){ - if(err) { - winston.error("Error creating index " + err.message); - } - }); - } - }); + db.createCollection('search', function(err, collection) { + if(err) { + winston.error("Error creating collection " + err.message); + return; + } + if(collection) { + collection.ensureIndex({content:'text'}, {background:true}, function(err, name){ + if(err) { + winston.error("Error creating index " + err.message); + } + }); + } + }); - callback(err); + callback(null); + } }); } diff --git a/src/upgrade.js b/src/upgrade.js index 99657007f7..75265c4643 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -309,7 +309,6 @@ Upgrade.upgradeMongo = function(callback) { async.series([ function(next) { db.get('schemaDate', function(err, value) { - console.log(schemaDate) schemaDate = value; thisSchemaDate = new Date(2013, 11, 6).getTime(); next(); From ab63ca6d925e3fec41aebb76044e4ef1975588ec Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Fri, 6 Dec 2013 14:53:03 -0500 Subject: [PATCH 78/83] store strings in sorted sets and lists to mimic redis --- src/database/mongo.js | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/database/mongo.js b/src/database/mongo.js index e9cfaa68e5..9408aff75a 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -497,7 +497,9 @@ // sorted sets module.sortedSetAdd = function(key, score, value, callback) { - + if(value !== null && value !== undefined) { + value = value.toString(); + } var data = { score:score, value:value @@ -508,6 +510,9 @@ } module.sortedSetRemove = function(key, value, callback) { + if(value !== null && value !== undefined) { + value = value.toString(); + } db.collection('objects').remove({setName:key, value:value}, function(err, result) { if(callback) { callback(err, result); @@ -584,6 +589,9 @@ } module.sortedSetRank = function(key, value, callback) { + if(value !== null && value !== undefined) { + value = value.toString(); + } module.getSortedSetRange(key, 0, -1, function(err, result) { if(err) { return callback(err); @@ -598,6 +606,9 @@ } module.sortedSetScore = function(key, value, callback) { + if(value !== null && value !== undefined) { + value = value.toString(); + } db.collection('objects').findOne({setName:key, value: value}, {fields:{score:1}}, function(err, result) { if(err) { return callback(err); @@ -611,6 +622,9 @@ } module.sortedSetsScore = function(keys, value, callback) { + if(value !== null && value !== undefined) { + value = value.toString(); + } db.collection('objects').find({setName:{$in:keys}, value: value}).toArray(function(err, result) { if(err) { return callback(err); @@ -635,6 +649,9 @@ // lists module.listPrepend = function(key, value, callback) { + if(value !== null && value !== undefined) { + value = value.toString(); + } module.isObjectField(key, 'array', function(err, exists) { if(err) { if(callback) { @@ -658,6 +675,9 @@ } module.listAppend = function(key, value, callback) { + if(value !== null && value !== undefined) { + value = value.toString(); + } db.collection('objects').update({ _key: key }, { $push: { array: value } }, {upsert:true, w:1}, function(err, result) { if(callback) { callback(err, result); From 9ca10c25d447e3024bfe9ddf23a1d53f977f1783 Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Fri, 6 Dec 2013 15:23:48 -0500 Subject: [PATCH 79/83] fixed return in posts.create --- src/posts.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/posts.js b/src/posts.js index df7557b874..c496afdd99 100644 --- a/src/posts.js +++ b/src/posts.js @@ -21,15 +21,14 @@ var db = require('./database'), Posts.create = function(uid, tid, content, callback) { if (uid === null) { - callback(new Error('invalid-user'), null); - return; + return callback(new Error('invalid-user'), null); } topics.isLocked(tid, function(err, locked) { if(err) { return callback(err, null); } else if(locked) { - callback(new Error('topic-locked'), null); + return callback(new Error('topic-locked'), null); } db.incrObjectField('global', 'nextPid', function(err, pid) { From 9ee250b59791ab9d7c37aafe73545d58b01d022d Mon Sep 17 00:00:00 2001 From: Baris Usakli Date: Fri, 6 Dec 2013 16:26:07 -0500 Subject: [PATCH 80/83] fixes humanreadable numbers in infinite scrolling --- public/src/forum/category.js | 3 ++- public/src/forum/recent.js | 3 ++- public/src/forum/unread.js | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/public/src/forum/category.js b/public/src/forum/category.js index 77daf9994c..44a11eaea9 100644 --- a/public/src/forum/category.js +++ b/public/src/forum/category.js @@ -140,10 +140,11 @@ define(function () { jQuery('#topics-container, .category-sidebar').removeClass('hidden'); jQuery('#category-no-topics').remove(); + html = $(html); container.append(html); $('#topics-container span.timeago').timeago(); - app.makeNumbersHumanReadable($(html).find('.human-readable-number')); + app.makeNumbersHumanReadable(html.find('.human-readable-number')); } Category.loadMoreTopics = function(cid) { diff --git a/public/src/forum/recent.js b/public/src/forum/recent.js index 4614c2f404..6b162db34f 100644 --- a/public/src/forum/recent.js +++ b/public/src/forum/recent.js @@ -89,9 +89,10 @@ define(function() { $('#category-no-topics').remove(); + html = $(html); container.append(html); $('span.timeago').timeago(); - app.makeNumbersHumanReadable($(html).find('.human-readable-number')); + app.makeNumbersHumanReadable(html.find('.human-readable-number')); } Recent.loadMoreTopics = function() { diff --git a/public/src/forum/unread.js b/public/src/forum/unread.js index 60c4ba4953..6debeeba84 100644 --- a/public/src/forum/unread.js +++ b/public/src/forum/unread.js @@ -79,9 +79,10 @@ define(function() { $('#category-no-topics').remove(); + html = $(html); container.append(html); $('span.timeago').timeago(); - app.makeNumbersHumanReadable($(html).find('.human-readable-number')); + app.makeNumbersHumanReadable(html.find('.human-readable-number')); } function loadMoreTopics() { From 8f769d53a3be8ddad3dde111e79a89ca050c62e5 Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Fri, 6 Dec 2013 17:46:03 -0500 Subject: [PATCH 81/83] fixed missing winston --- src/database.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/database.js b/src/database.js index 47c76c30c1..ba52b99d52 100644 --- a/src/database.js +++ b/src/database.js @@ -1,12 +1,13 @@ var nconf = require('nconf'), - databaseType = nconf.get('database'); + databaseType = nconf.get('database'), + winston = require('winston'); - if(!databaseType) { - winston.info('Database type not set! Run npm app --setup'); - process.exit(); - } +if(!databaseType) { + winston.info('Database type not set! Run npm app --setup'); + process.exit(); +} var db = require('./database/' + databaseType); From 2a21b4855efe6d15440e7634e82d88ca87c9cb4b Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Fri, 6 Dec 2013 17:47:48 -0500 Subject: [PATCH 82/83] fixed instruction in database.js --- src/database.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database.js b/src/database.js index ba52b99d52..71775e8807 100644 --- a/src/database.js +++ b/src/database.js @@ -5,7 +5,7 @@ var nconf = require('nconf'), winston = require('winston'); if(!databaseType) { - winston.info('Database type not set! Run npm app --setup'); + winston.info('Database type not set! Run node app --setup'); process.exit(); } From 90a75ee0452bda5aa1a377f7028d22c3e6b8001c Mon Sep 17 00:00:00 2001 From: Baris Soner Usakli Date: Fri, 6 Dec 2013 17:51:16 -0500 Subject: [PATCH 83/83] added check for unknown db --- src/install.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/install.js b/src/install.js index 4deca311cd..601fdff077 100644 --- a/src/install.js +++ b/src/install.js @@ -110,7 +110,7 @@ var async = require('async'), } function dbQuestionsSuccess(err, databaseConfig) { - if (!config) { + if (!databaseConfig) { return next(new Error('aborted')); } @@ -129,9 +129,10 @@ var async = require('async'), password: databaseConfig['mongo:password'], database: databaseConfig['mongo:database'] }; + } else { + return next(new Error('unknown database : ' + config.database)); } - // Add hardcoded values config.bcrypt_rounds = 12; config.upload_path = '/public/uploads'; config.use_port = (config.use_port.slice(0, 1) === 'y') ? true : false; @@ -153,10 +154,13 @@ var async = require('async'), }); } - if(config.database === 'redis') + if(config.database === 'redis') { prompt.get(install.redisQuestions, dbQuestionsSuccess); - else if(config.database === 'mongo') + } else if(config.database === 'mongo') { prompt.get(install.mongoQuestions, dbQuestionsSuccess); + } else { + return next(new Error('unknown database : ' + config.database)); + } }; // prompt prepends "prompt: " to questions, let's clear that.