diff --git a/src/controllers/admin/cache.js b/src/controllers/admin/cache.js index 1284be08be..a3ce7fba21 100644 --- a/src/controllers/admin/cache.js +++ b/src/controllers/admin/cache.js @@ -5,7 +5,7 @@ var cacheController = module.exports; cacheController.get = function (req, res) { var postCache = require('../../posts/cache'); var groupCache = require('../../groups').cache; - var userSettingsCache = require('../../user').settingsCache; + var objectCache = require('../../database').objectCache; var avgPostSize = 0; var percentFull = 0; @@ -14,7 +14,7 @@ cacheController.get = function (req, res) { percentFull = ((postCache.length / postCache.max) * 100).toFixed(2); } - res.render('admin/advanced/cache', { + var data = { postCache: { length: postCache.length, max: postCache.max, @@ -22,12 +22,6 @@ cacheController.get = function (req, res) { percentFull: percentFull, avgPostSize: avgPostSize, }, - userSettingsCache: { - length: userSettingsCache.length, - max: userSettingsCache.max, - itemCount: userSettingsCache.itemCount, - percentFull: ((userSettingsCache.length / userSettingsCache.max) * 100).toFixed(2), - }, groupCache: { length: groupCache.length, max: groupCache.max, @@ -35,5 +29,17 @@ cacheController.get = function (req, res) { percentFull: ((groupCache.length / groupCache.max) * 100).toFixed(2), dump: req.query.debug ? JSON.stringify(groupCache.dump(), null, 4) : false, }, - }); + }; + + if (objectCache) { + data.objectCache = { + length: objectCache.length, + max: objectCache.max, + itemCount: objectCache.itemCount, + percentFull: ((objectCache.length / objectCache.max) * 100).toFixed(2), + dump: req.query.debug ? JSON.stringify(objectCache.dump(), null, 4) : false, + }; + } + + res.render('admin/advanced/cache', data); }; diff --git a/src/database/mongo/hash.js b/src/database/mongo/hash.js index 997f22cfc0..4985afec2f 100644 --- a/src/database/mongo/hash.js +++ b/src/database/mongo/hash.js @@ -3,6 +3,36 @@ module.exports = function (db, module) { var helpers = module.helpers.mongo; + var LRU = require('lru-cache'); + var _ = require('lodash'); + var pubsub = require('../../pubsub'); + + var cache = LRU({ + max: 10000, + length: function () { return 1; }, + maxAge: 0, + }); + + module.objectCache = cache; + + pubsub.on('mongo:hash:cache:del', function (key) { + cache.del(key); + }); + + pubsub.on('mongo:hash:cache:reset', function () { + cache.reset(); + }); + + module.delObjectCache = function (key) { + pubsub.publish('mongo:hash:cache:del', key); + cache.del(key); + }; + + module.resetObjectCache = function () { + pubsub.publish('mongo:hash:cache:reset'); + cache.reset(); + }; + module.setObject = function (key, data, callback) { callback = callback || helpers.noop; if (!key || !data) { @@ -12,7 +42,11 @@ module.exports = function (db, module) { delete data['']; } db.collection('objects').update({ _key: key }, { $set: data }, { upsert: true, w: 1 }, function (err) { - callback(err); + if (err) { + return callback(err); + } + module.delObjectCache(key); + callback(); }); }; @@ -31,26 +65,48 @@ module.exports = function (db, module) { if (!key) { return callback(); } - db.collection('objects').findOne({ _key: key }, { _id: 0, _key: 0 }, callback); + + module.getObjects([key], function (err, data) { + if (err) { + return callback(err); + } + callback(null, data && data.length ? data[0] : null); + }); }; module.getObjects = function (keys, callback) { + function getFromCache(next) { + setImmediate(next, null, keys.map(function (key) { + return _.clone(cache.get(key)); + })); + } + if (!Array.isArray(keys) || !keys.length) { return callback(null, []); } - db.collection('objects').find({ _key: { $in: keys } }, { _id: 0 }).toArray(function (err, data) { + + var nonCachedKeys = keys.filter(function (key) { + return !cache.get(key); + }); + + if (!nonCachedKeys.length) { + return getFromCache(callback); + } + + db.collection('objects').find({ _key: { $in: nonCachedKeys } }, { _id: 0 }).toArray(function (err, data) { if (err) { return callback(err); } - var map = helpers.toMap(data); - var returnData = []; - - for (var i = 0; i < keys.length; i += 1) { - returnData.push(map[keys[i]]); - } + data.forEach(function (objectData) { + if (objectData) { + var key = objectData._key; + delete objectData._key; + cache.set(key, objectData); + } + }); - callback(null, returnData); + getFromCache(callback); }); }; @@ -58,16 +114,10 @@ module.exports = function (db, module) { if (!key) { return callback(); } - field = helpers.fieldToString(field); - var _fields = { - _id: 0, - }; - _fields[field] = 1; - db.collection('objects').findOne({ _key: key }, { fields: _fields }, function (err, item) { + module.getObject(key, function (err, item) { if (err || !item) { return callback(err, null); } - callback(null, item.hasOwnProperty(field) ? item[field] : null); }); }; @@ -76,22 +126,13 @@ module.exports = function (db, module) { if (!key) { return callback(); } - var _fields = { - _id: 0, - }; - var i; - - for (i = 0; i < fields.length; i += 1) { - fields[i] = helpers.fieldToString(fields[i]); - _fields[fields[i]] = 1; - } - db.collection('objects').findOne({ _key: key }, { fields: _fields }, function (err, item) { + module.getObject(key, function (err, item) { if (err) { return callback(err); } item = item || {}; var result = {}; - for (i = 0; i < fields.length; i += 1) { + for (var i = 0; i < fields.length; i += 1) { result[fields[i]] = item[fields[i]] !== undefined ? item[fields[i]] : null; } callback(null, result); @@ -102,38 +143,24 @@ module.exports = function (db, module) { if (!Array.isArray(keys) || !keys.length) { return callback(null, []); } - var _fields = { - _id: 0, - _key: 1, - }; - - for (var i = 0; i < fields.length; i += 1) { - fields[i] = helpers.fieldToString(fields[i]); - _fields[fields[i]] = 1; - } - - db.collection('objects').find({ _key: { $in: keys } }, { fields: _fields }).toArray(function (err, items) { + module.getObjects(keys, function (err, items) { if (err) { return callback(err); } - if (items === null) { items = []; } - var map = helpers.toMap(items); var returnData = []; var item; - + var result; for (var i = 0; i < keys.length; i += 1) { - item = map[keys[i]] || {}; - + item = items[i] || {}; + result = {}; for (var k = 0; k < fields.length; k += 1) { - if (item[fields[k]] === undefined) { - item[fields[k]] = null; - } + result[fields[k]] = item[fields[k]] !== undefined ? item[fields[k]] : null; } - returnData.push(item); + returnData.push(result); } callback(null, returnData); @@ -220,7 +247,11 @@ module.exports = function (db, module) { }); db.collection('objects').update({ _key: key }, { $unset: data }, function (err) { - callback(err); + if (err) { + return callback(err); + } + module.delObjectCache(key); + callback(); }); }; @@ -244,7 +275,11 @@ module.exports = function (db, module) { data[field] = value; db.collection('objects').findAndModify({ _key: key }, {}, { $inc: data }, { new: true, upsert: true }, function (err, result) { - callback(err, result && result.value ? result.value[field] : null); + if (err) { + return callback(err); + } + module.delObjectCache(key); + callback(null, result && result.value ? result.value[field] : null); }); }; }; diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js index 134e6f801d..eff7459a69 100644 --- a/src/database/mongo/main.js +++ b/src/database/mongo/main.js @@ -13,7 +13,11 @@ module.exports = function (db, module) { module.emptydb = function (callback) { callback = callback || helpers.noop; db.collection('objects').remove({}, function (err) { - callback(err); + if (err) { + return callback(err); + } + module.resetObjectCache(); + callback(); }); }; @@ -32,7 +36,11 @@ module.exports = function (db, module) { return callback(); } db.collection('objects').remove({ _key: key }, function (err) { - callback(err); + if (err) { + return callback(err); + } + module.delObjectCache(key); + callback(); }); }; @@ -42,7 +50,15 @@ module.exports = function (db, module) { return callback(); } db.collection('objects').remove({ _key: { $in: keys } }, function (err) { - callback(err); + if (err) { + return callback(err); + } + + keys.forEach(function (key) { + module.delObjectCache(key); + }); + + callback(null); }); }; @@ -75,7 +91,12 @@ module.exports = function (db, module) { module.rename = function (oldKey, newKey, callback) { callback = callback || helpers.noop; db.collection('objects').update({ _key: oldKey }, { $set: { _key: newKey } }, { multi: true }, function (err) { - callback(err); + if (err) { + return callback(err); + } + module.delObjectCache(oldKey); + module.delObjectCache(newKey); + callback(); }); }; diff --git a/src/database/mongo/sets.js b/src/database/mongo/sets.js index 18ce2a932a..6d6abbfbab 100644 --- a/src/database/mongo/sets.js +++ b/src/database/mongo/sets.js @@ -137,13 +137,13 @@ module.exports = function (db, module) { if (err) { return callback(err); } - - result = result.map(function (item) { - return item._key; + var map = {}; + result.forEach(function (item) { + map[item._key] = true; }); result = sets.map(function (set) { - return result.indexOf(set) !== -1; + return !!map[set]; }); callback(null, result); diff --git a/src/groups/membership.js b/src/groups/membership.js index 4d4a1f2887..42c3932805 100644 --- a/src/groups/membership.js +++ b/src/groups/membership.js @@ -15,7 +15,7 @@ var LRU = require('lru-cache'); var cache = LRU({ max: 40000, - maxAge: 1000 * 60 * 60, + maxAge: 0, }); module.exports = function (Groups) { @@ -382,7 +382,7 @@ module.exports = function (Groups) { } var nonCachedUids = uids.filter(function (uid) { - return !cache.has(uid + ':' + groupName); + return !cache.get(uid + ':' + groupName); }); if (!nonCachedUids.length) { @@ -415,7 +415,7 @@ module.exports = function (Groups) { } var nonCachedGroups = groups.filter(function (groupName) { - return !cache.has(uid + ':' + groupName); + return !cache.get(uid + ':' + groupName); }); if (!nonCachedGroups.length) { diff --git a/src/meta/blacklist.js b/src/meta/blacklist.js index d872f3ee07..6fa657c5e8 100644 --- a/src/meta/blacklist.js +++ b/src/meta/blacklist.js @@ -44,8 +44,8 @@ Blacklist.save = function (rules, callback) { db.setObject('ip-blacklist-rules', { rules: rules }, next); }, function (next) { + Blacklist.load(next); pubsub.publish('blacklist:reload'); - setImmediate(next); }, ], callback); }; diff --git a/src/posts/cache.js b/src/posts/cache.js index ae4f1948a6..9fce761490 100644 --- a/src/posts/cache.js +++ b/src/posts/cache.js @@ -6,7 +6,7 @@ var meta = require('../meta'); var cache = LRU({ max: parseInt(meta.config.postCacheSize, 10) || 1048576, length: function (n) { return n.length; }, - maxAge: 1000 * 60 * 60, + maxAge: 0, }); module.exports = cache; diff --git a/src/user/settings.js b/src/user/settings.js index 1a42222acf..6efbac67f3 100644 --- a/src/user/settings.js +++ b/src/user/settings.js @@ -2,38 +2,17 @@ 'use strict'; var async = require('async'); -var _ = require('lodash'); var meta = require('../meta'); var db = require('../database'); var plugins = require('../plugins'); -var pubsub = require('../pubsub'); -var LRU = require('lru-cache'); - -var cache = LRU({ - max: 2000, - length: function () { return 1; }, - maxAge: 1000 * 60 * 60, -}); - module.exports = function (User) { - User.settingsCache = cache; - - pubsub.on('user:settings:cache:del', function (uid) { - cache.del('user:' + uid + ':settings'); - }); - User.getSettings = function (uid, callback) { if (!parseInt(uid, 10)) { return onSettingsLoaded(0, {}, callback); } - var cached = cache.get('user:' + uid + ':settings'); - if (cached) { - return onSettingsLoaded(uid, _.clone(cached || {}), callback); - } - async.waterfall([ function (next) { db.getObject('user:' + uid + ':settings', next); @@ -41,36 +20,17 @@ module.exports = function (User) { function (settings, next) { settings = settings || {}; settings.uid = uid; - cache.set('user:' + uid + ':settings', settings); - onSettingsLoaded(uid, _.clone(settings || {}), next); + onSettingsLoaded(uid, settings, next); }, ], callback); }; User.getMultipleUserSettings = function (uids, callback) { - function getFromCache(next) { - var settings = uids.map(function (uid) { - return cache.get('user:' + uid + ':settings') || {}; - }); - async.map(settings, function (setting, next) { - onSettingsLoaded(setting.uid, _.clone(setting), next); - }, next); - } - - if (!Array.isArray(uids) || !uids.length) { return callback(null, []); } - var nonCachedUids = uids.filter(function (uid) { - return !cache.has('user:' + uid + ':settings'); - }); - - if (!nonCachedUids.length) { - return getFromCache(callback); - } - - var keys = nonCachedUids.map(function (uid) { + var keys = uids.map(function (uid) { return 'user:' + uid + ':settings'; }); @@ -79,13 +39,13 @@ module.exports = function (User) { db.getObjects(keys, next); }, function (settings, next) { - settings.forEach(function (userSettings, index) { + settings = settings.map(function (userSettings, index) { userSettings = userSettings || {}; - userSettings.uid = nonCachedUids[index]; - cache.set('user:' + userSettings.uid + ':settings', userSettings); + userSettings.uid = uids[index]; + return userSettings; }); - getFromCache(next); + next(null, settings); }, ], callback); }; @@ -187,8 +147,6 @@ module.exports = function (User) { User.updateDigestSetting(uid, data.dailyDigestFreq, next); }, function (next) { - cache.del('user:' + uid + ':settings'); - pubsub.publish('user:settings:cache:del', uid); User.getSettings(uid, next); }, ], callback); @@ -213,7 +171,7 @@ module.exports = function (User) { if (!parseInt(uid, 10)) { return setImmediate(callback); } - cache.del('user:' + uid + ':settings'); + db.setObjectField('user:' + uid + ':settings', key, value, callback); }; }; diff --git a/src/views/admin/advanced/cache.tpl b/src/views/admin/advanced/cache.tpl index 72854a34b0..1ea4aa39d9 100644 --- a/src/views/admin/advanced/cache.tpl +++ b/src/views/admin/advanced/cache.tpl @@ -27,20 +27,21 @@
-
User Settings Cache
+
Object Cache
-
- {userSettingsCache.itemCount}
-
- {userSettingsCache.length} / {userSettingsCache.max}
+ {objectCache.length} / {objectCache.max}
-
- [[admin/advanced/cache:percent-full, {userSettingsCache.percentFull}]] +
+ [[admin/advanced/cache:percent-full, {objectCache.percentFull}]]
+ + + 
{objectCache.dump}
+
@@ -48,9 +49,6 @@
Group Cache
-
- {groupCache.itemCount}
-
{groupCache.length} / {groupCache.max}
diff --git a/test/database/hash.js b/test/database/hash.js index 7b128d76f1..0f8f3d0cc6 100644 --- a/test/database/hash.js +++ b/test/database/hash.js @@ -19,8 +19,8 @@ describe('Hash methods', function () { describe('setObject()', function () { it('should create a object', function (done) { db.setObject('testObject1', { foo: 'baris', bar: 99 }, function (err) { - assert.equal(err, null); - assert.equal(arguments.length, 1); + assert.ifError(err); + assert(arguments.length < 2); done(); }); }); @@ -57,16 +57,16 @@ describe('Hash methods', function () { describe('setObjectField()', function () { it('should create a new object with field', function (done) { db.setObjectField('testObject2', 'name', 'ginger', function (err) { - assert.equal(err, null); - assert.equal(arguments.length, 1); + assert.ifError(err); + assert(arguments.length < 2); done(); }); }); it('should add a new field to an object', function (done) { db.setObjectField('testObject2', 'type', 'cat', function (err) { - assert.equal(err, null); - assert.equal(arguments.length, 1); + assert.ifError(err, null); + assert(arguments.length < 2); done(); }); }); @@ -305,10 +305,10 @@ describe('Hash methods', function () { it('should delete an objects field', function (done) { db.deleteObjectField('testObject10', 'delete', function (err) { - assert.equal(err, null); - assert.equal(arguments.length, 1); + assert.ifError(err); + assert(arguments.length < 2); db.isObjectField('testObject10', 'delete', function (err, isField) { - assert.equal(err, null); + assert.ifError(err); assert.equal(isField, false); done(); }); @@ -318,7 +318,7 @@ describe('Hash methods', function () { it('should delete multiple fields of the object', function (done) { db.deleteObjectFields('testObject10', ['delete1', 'delete2'], function (err) { assert.ifError(err); - assert.equal(arguments.length, 1); + assert(arguments.length < 2); async.parallel({ delete1: async.apply(db.isObjectField, 'testObject10', 'delete1'), delete2: async.apply(db.isObjectField, 'testObject10', 'delete2'), diff --git a/test/database/keys.js b/test/database/keys.js index 330d84bdd9..03d607d808 100644 --- a/test/database/keys.js +++ b/test/database/keys.js @@ -12,15 +12,15 @@ describe('Key methods', function () { it('should set a key without error', function (done) { db.set('testKey', 'testValue', function (err) { - assert.equal(err, null); - assert.equal(arguments.length, 1); + assert.ifError(err); + assert(arguments.length < 2); done(); }); }); it('should get a key without error', function (done) { db.get('testKey', function (err, value) { - assert.equal(err, null); + assert.ifError(err); assert.equal(arguments.length, 2); assert.strictEqual(value, 'testValue'); done(); @@ -29,7 +29,7 @@ describe('Key methods', function () { it('should return true if key exist', function (done) { db.exists('testKey', function (err, exists) { - assert.equal(err, null); + assert.ifError(err); assert.equal(arguments.length, 2); assert.strictEqual(exists, true); done(); @@ -38,7 +38,7 @@ describe('Key methods', function () { it('should return false if key does not exist', function (done) { db.exists('doesnotexist', function (err, exists) { - assert.equal(err, null); + assert.ifError(err); assert.equal(arguments.length, 2); assert.strictEqual(exists, false); done(); @@ -47,11 +47,11 @@ describe('Key methods', function () { it('should delete a key without error', function (done) { db.delete('testKey', function (err) { - assert.equal(err, null); - assert.equal(arguments.length, 1); + assert.ifError(err); + assert(arguments.length < 2); db.get('testKey', function (err, value) { - assert.equal(err, null); + assert.ifError(err); assert.equal(false, !!value); done(); }); @@ -60,10 +60,10 @@ describe('Key methods', function () { it('should return false if key was deleted', function (done) { db.delete('testKey', function (err) { - assert.equal(err, null); - assert.equal(arguments.length, 1); + assert.ifError(err); + assert(arguments.length < 2); db.exists('testKey', function (err, exists) { - assert.equal(err, null); + assert.ifError(err); assert.strictEqual(exists, false); done(); }); @@ -83,7 +83,7 @@ describe('Key methods', function () { return done(err); } db.deleteAll(['key1', 'key2'], function (err) { - assert.equal(err, null); + assert.ifError(err); assert.equal(arguments.length, 1); async.parallel({ key1exists: function (next) { @@ -93,7 +93,7 @@ describe('Key methods', function () { db.exists('key2', next); }, }, function (err, results) { - assert.equal(err, null); + assert.ifError(err); assert.equal(results.key1exists, false); assert.equal(results.key2exists, false); done(); @@ -124,7 +124,7 @@ describe('Key methods', function () { db.isSortedSetMember('deletezset', 'value2', next); }, }, function (err, results) { - assert.equal(err, null); + assert.ifError(err); assert.equal(results.key1exists, false); assert.equal(results.key2exists, false); done(); @@ -136,7 +136,7 @@ describe('Key methods', function () { describe('increment', function () { it('should initialize key to 1', function (done) { db.increment('keyToIncrement', function (err, value) { - assert.equal(err, null); + assert.ifError(err); assert.strictEqual(parseInt(value, 10), 1); done(); }); @@ -144,7 +144,7 @@ describe('Key methods', function () { it('should increment key to 2', function (done) { db.increment('keyToIncrement', function (err, value) { - assert.equal(err, null); + assert.ifError(err); assert.strictEqual(parseInt(value, 10), 2); done(); }); @@ -158,11 +158,11 @@ describe('Key methods', function () { return done(err); } db.rename('keyOldName', 'keyNewName', function (err) { - assert.equal(err, null); - assert.equal(arguments.length, 1); + assert.ifError(err); + assert(arguments.length < 2); db.get('keyNewName', function (err, value) { - assert.equal(err, null); + assert.ifError(err); assert.equal(value, 'renamedKeyValue'); done(); });