Mongo cache (#6002)

* cache mongodb objects

remove user settings cache since it is cached at the object level

* removed indexOf from isMemberOfSets

* change has to get, change maxAge to 0

src/controllers/admin/cache.js

@@ -5,7 +5,7 @@ var cacheController = module.exports;
 cacheController.get = function (req, res) {
 	var postCache = require('../../posts/cache');
 	var groupCache = require('../../groups').cache;
-	var userSettingsCache = require('../../user').settingsCache;
+	var objectCache = require('../../database').objectCache;
 
 	var avgPostSize = 0;
 	var percentFull = 0;
@@ -14,7 +14,7 @@ cacheController.get = function (req, res) {
 		percentFull = ((postCache.length / postCache.max) * 100).toFixed(2);
 	}
 
-	res.render('admin/advanced/cache', {
+	var data = {
 		postCache: {
 			length: postCache.length,
 			max: postCache.max,
@@ -22,12 +22,6 @@ cacheController.get = function (req, res) {
 			percentFull: percentFull,
 			avgPostSize: avgPostSize,
 		},
-		userSettingsCache: {
-			length: userSettingsCache.length,
-			max: userSettingsCache.max,
-			itemCount: userSettingsCache.itemCount,
-			percentFull: ((userSettingsCache.length / userSettingsCache.max) * 100).toFixed(2),
-		},
 		groupCache: {
 			length: groupCache.length,
 			max: groupCache.max,
@@ -35,5 +29,17 @@ cacheController.get = function (req, res) {
 			percentFull: ((groupCache.length / groupCache.max) * 100).toFixed(2),
 			dump: req.query.debug ? JSON.stringify(groupCache.dump(), null, 4) : false,
 		},
-	});
+	};
+
+	if (objectCache) {
+		data.objectCache = {
+			length: objectCache.length,
+			max: objectCache.max,
+			itemCount: objectCache.itemCount,
+			percentFull: ((objectCache.length / objectCache.max) * 100).toFixed(2),
+			dump: req.query.debug ? JSON.stringify(objectCache.dump(), null, 4) : false,
+		};
+	}
+
+	res.render('admin/advanced/cache', data);
 };

src/database/mongo/hash.js

@@ -3,6 +3,36 @@
 module.exports = function (db, module) {
 	var helpers = module.helpers.mongo;
 
+	var LRU = require('lru-cache');
+	var _ = require('lodash');
+	var pubsub = require('../../pubsub');
+
+	var cache = LRU({
+		max: 10000,
+		length: function () { return 1; },
+		maxAge: 0,
+	});
+
+	module.objectCache = cache;
+
+	pubsub.on('mongo:hash:cache:del', function (key) {
+		cache.del(key);
+	});
+
+	pubsub.on('mongo:hash:cache:reset', function () {
+		cache.reset();
+	});
+
+	module.delObjectCache = function (key) {
+		pubsub.publish('mongo:hash:cache:del', key);
+		cache.del(key);
+	};
+
+	module.resetObjectCache = function () {
+		pubsub.publish('mongo:hash:cache:reset');
+		cache.reset();
+	};
+
 	module.setObject = function (key, data, callback) {
 		callback = callback || helpers.noop;
 		if (!key || !data) {
@@ -12,7 +42,11 @@ module.exports = function (db, module) {
 			delete data[''];
 		}
 		db.collection('objects').update({ _key: key }, { $set: data }, { upsert: true, w: 1 }, function (err) {
-			callback(err);
+			if (err) {
+				return callback(err);
+			}
+			module.delObjectCache(key);
+			callback();
 		});
 	};
 
@@ -31,26 +65,48 @@ module.exports = function (db, module) {
 		if (!key) {
 			return callback();
 		}
-		db.collection('objects').findOne({ _key: key }, { _id: 0, _key: 0 }, callback);
+
+		module.getObjects([key], function (err, data) {
+			if (err) {
+				return callback(err);
+			}
+			callback(null, data && data.length ? data[0] : null);
+		});
 	};
 
 	module.getObjects = function (keys, callback) {
+		function getFromCache(next) {
+			setImmediate(next, null, keys.map(function (key) {
+				return _.clone(cache.get(key));
+			}));
+		}
+
 		if (!Array.isArray(keys) || !keys.length) {
 			return callback(null, []);
 		}
-		db.collection('objects').find({ _key: { $in: keys } }, { _id: 0 }).toArray(function (err, data) {
+
+		var nonCachedKeys = keys.filter(function (key) {
+			return !cache.get(key);
+		});
+
+		if (!nonCachedKeys.length) {
+			return getFromCache(callback);
+		}
+
+		db.collection('objects').find({ _key: { $in: nonCachedKeys } }, { _id: 0 }).toArray(function (err, data) {
 			if (err) {
 				return callback(err);
 			}
 
-			var map = helpers.toMap(data);
-			var returnData = [];
-
-			for (var i = 0; i < keys.length; i += 1) {
-				returnData.push(map[keys[i]]);
-			}
+			data.forEach(function (objectData) {
+				if (objectData) {
+					var key = objectData._key;
+					delete objectData._key;
+					cache.set(key, objectData);
+				}
+			});
 
-			callback(null, returnData);
+			getFromCache(callback);
 		});
 	};
 
@@ -58,16 +114,10 @@ module.exports = function (db, module) {
 		if (!key) {
 			return callback();
 		}
-		field = helpers.fieldToString(field);
-		var _fields = {
-			_id: 0,
-		};
-		_fields[field] = 1;
-		db.collection('objects').findOne({ _key: key }, { fields: _fields }, function (err, item) {
+		module.getObject(key, function (err, item) {
 			if (err || !item) {
 				return callback(err, null);
 			}
-
 			callback(null, item.hasOwnProperty(field) ? item[field] : null);
 		});
 	};
@@ -76,22 +126,13 @@ module.exports = function (db, module) {
 		if (!key) {
 			return callback();
 		}
-		var _fields = {
-			_id: 0,
-		};
-		var i;
-
-		for (i = 0; i < fields.length; i += 1) {
-			fields[i] = helpers.fieldToString(fields[i]);
-			_fields[fields[i]] = 1;
-		}
-		db.collection('objects').findOne({ _key: key }, { fields: _fields }, function (err, item) {
+		module.getObject(key, function (err, item) {
 			if (err) {
 				return callback(err);
 			}
 			item = item || {};
 			var result = {};
-			for (i = 0; i < fields.length; i += 1) {
+			for (var i = 0; i < fields.length; i += 1) {
 				result[fields[i]] = item[fields[i]] !== undefined ? item[fields[i]] : null;
 			}
 			callback(null, result);
@@ -102,38 +143,24 @@ module.exports = function (db, module) {
 		if (!Array.isArray(keys) || !keys.length) {
 			return callback(null, []);
 		}
-		var _fields = {
-			_id: 0,
-			_key: 1,
-		};
-
-		for (var i = 0; i < fields.length; i += 1) {
-			fields[i] = helpers.fieldToString(fields[i]);
-			_fields[fields[i]] = 1;
-		}
-
-		db.collection('objects').find({ _key: { $in: keys } }, { fields: _fields }).toArray(function (err, items) {
+		module.getObjects(keys, function (err, items) {
 			if (err) {
 				return callback(err);
 			}
-
 			if (items === null) {
 				items = [];
 			}
 
-			var map = helpers.toMap(items);
 			var returnData = [];
 			var item;
-
+			var result;
 			for (var i = 0; i < keys.length; i += 1) {
-				item = map[keys[i]] || {};
-
+				item = items[i] || {};
+				result = {};
 				for (var k = 0; k < fields.length; k += 1) {
-					if (item[fields[k]] === undefined) {
-						item[fields[k]] = null;
-					}
+					result[fields[k]] = item[fields[k]] !== undefined ? item[fields[k]] : null;
 				}
-				returnData.push(item);
+				returnData.push(result);
 			}
 
 			callback(null, returnData);
@@ -220,7 +247,11 @@ module.exports = function (db, module) {
 		});
 
 		db.collection('objects').update({ _key: key }, { $unset: data }, function (err) {
-			callback(err);
+			if (err) {
+				return callback(err);
+			}
+			module.delObjectCache(key);
+			callback();
 		});
 	};
 
@@ -244,7 +275,11 @@ module.exports = function (db, module) {
 		data[field] = value;
 
 		db.collection('objects').findAndModify({ _key: key }, {}, { $inc: data }, { new: true, upsert: true }, function (err, result) {
-			callback(err, result && result.value ? result.value[field] : null);
+			if (err) {
+				return callback(err);
+			}
+			module.delObjectCache(key);
+			callback(null, result && result.value ? result.value[field] : null);
 		});
 	};
 };

src/database/mongo/main.js

@@ -13,7 +13,11 @@ module.exports = function (db, module) {
 	module.emptydb = function (callback) {
 		callback = callback || helpers.noop;
 		db.collection('objects').remove({}, function (err) {
-			callback(err);
+			if (err) {
+				return callback(err);
+			}
+			module.resetObjectCache();
+			callback();
 		});
 	};
 
@@ -32,7 +36,11 @@ module.exports = function (db, module) {
 			return callback();
 		}
 		db.collection('objects').remove({ _key: key }, function (err) {
-			callback(err);
+			if (err) {
+				return callback(err);
+			}
+			module.delObjectCache(key);
+			callback();
 		});
 	};
 
@@ -42,7 +50,15 @@ module.exports = function (db, module) {
 			return callback();
 		}
 		db.collection('objects').remove({ _key: { $in: keys } }, function (err) {
-			callback(err);
+			if (err) {
+				return callback(err);
+			}
+
+			keys.forEach(function (key) {
+				module.delObjectCache(key);
+			});
+
+			callback(null);
 		});
 	};
 
@@ -75,7 +91,12 @@ module.exports = function (db, module) {
 	module.rename = function (oldKey, newKey, callback) {
 		callback = callback || helpers.noop;
 		db.collection('objects').update({ _key: oldKey }, { $set: { _key: newKey } }, { multi: true }, function (err) {
-			callback(err);
+			if (err) {
+				return callback(err);
+			}
+			module.delObjectCache(oldKey);
+			module.delObjectCache(newKey);
+			callback();
 		});
 	};
 

src/database/mongo/sets.js

@@ -137,13 +137,13 @@ module.exports = function (db, module) {
 			if (err) {
 				return callback(err);
 			}
-
-			result = result.map(function (item) {
-				return item._key;
+			var map = {};
+			result.forEach(function (item) {
+				map[item._key] = true;
 			});
 
 			result = sets.map(function (set) {
-				return result.indexOf(set) !== -1;
+				return !!map[set];
 			});
 
 			callback(null, result);

src/groups/membership.js

@@ -15,7 +15,7 @@ var LRU = require('lru-cache');
 
 var cache = LRU({
 	max: 40000,
-	maxAge: 1000 * 60 * 60,
+	maxAge: 0,
 });
 
 module.exports = function (Groups) {
@@ -382,7 +382,7 @@ module.exports = function (Groups) {
 		}
 
 		var nonCachedUids = uids.filter(function (uid) {
-			return !cache.has(uid + ':' + groupName);
+			return !cache.get(uid + ':' + groupName);
 		});
 
 		if (!nonCachedUids.length) {
@@ -415,7 +415,7 @@ module.exports = function (Groups) {
 		}
 
 		var nonCachedGroups = groups.filter(function (groupName) {
-			return !cache.has(uid + ':' + groupName);
+			return !cache.get(uid + ':' + groupName);
 		});
 
 		if (!nonCachedGroups.length) {

src/meta/blacklist.js

@@ -44,8 +44,8 @@ Blacklist.save = function (rules, callback) {
 			db.setObject('ip-blacklist-rules', { rules: rules }, next);
 		},
 		function (next) {
+			Blacklist.load(next);
 			pubsub.publish('blacklist:reload');
-			setImmediate(next);
 		},
 	], callback);
 };

src/posts/cache.js

@@ -6,7 +6,7 @@ var meta = require('../meta');
 var cache = LRU({
 	max: parseInt(meta.config.postCacheSize, 10) || 1048576,
 	length: function (n) { return n.length; },
-	maxAge: 1000 * 60 * 60,
+	maxAge: 0,
 });
 
 module.exports = cache;

src/user/settings.js

@@ -2,38 +2,17 @@
 'use strict';
 
 var async = require('async');
-var _ = require('lodash');
 
 var meta = require('../meta');
 var db = require('../database');
 var plugins = require('../plugins');
 
-var pubsub = require('../pubsub');
-var LRU = require('lru-cache');
-
-var cache = LRU({
-	max: 2000,
-	length: function () { return 1; },
-	maxAge: 1000 * 60 * 60,
-});
-
 module.exports = function (User) {
-	User.settingsCache = cache;
-
-	pubsub.on('user:settings:cache:del', function (uid) {
-		cache.del('user:' + uid + ':settings');
-	});
-
 	User.getSettings = function (uid, callback) {
 		if (!parseInt(uid, 10)) {
 			return onSettingsLoaded(0, {}, callback);
 		}
 
-		var cached = cache.get('user:' + uid + ':settings');
-		if (cached) {
-			return onSettingsLoaded(uid, _.clone(cached || {}), callback);
-		}
-
 		async.waterfall([
 			function (next) {
 				db.getObject('user:' + uid + ':settings', next);
@@ -41,36 +20,17 @@ module.exports = function (User) {
 			function (settings, next) {
 				settings = settings || {};
 				settings.uid = uid;
-				cache.set('user:' + uid + ':settings', settings);
-				onSettingsLoaded(uid, _.clone(settings || {}), next);
+				onSettingsLoaded(uid, settings, next);
 			},
 		], callback);
 	};
 
 	User.getMultipleUserSettings = function (uids, callback) {
-		function getFromCache(next) {
-			var settings = uids.map(function (uid) {
-				return cache.get('user:' + uid + ':settings') || {};
-			});
-			async.map(settings, function (setting, next) {
-				onSettingsLoaded(setting.uid, _.clone(setting), next);
-			}, next);
-		}
-
-
 		if (!Array.isArray(uids) || !uids.length) {
 			return callback(null, []);
 		}
 
-		var nonCachedUids = uids.filter(function (uid) {
-			return !cache.has('user:' + uid + ':settings');
-		});
-
-		if (!nonCachedUids.length) {
-			return getFromCache(callback);
-		}
-
-		var keys = nonCachedUids.map(function (uid) {
+		var keys = uids.map(function (uid) {
 			return 'user:' + uid + ':settings';
 		});
 
@@ -79,13 +39,13 @@ module.exports = function (User) {
 				db.getObjects(keys, next);
 			},
 			function (settings, next) {
-				settings.forEach(function (userSettings, index) {
+				settings = settings.map(function (userSettings, index) {
 					userSettings = userSettings || {};
-					userSettings.uid = nonCachedUids[index];
-					cache.set('user:' + userSettings.uid + ':settings', userSettings);
+					userSettings.uid = uids[index];
+					return userSettings;
 				});
 
-				getFromCache(next);
+				next(null, settings);
 			},
 		], callback);
 	};
@@ -187,8 +147,6 @@ module.exports = function (User) {
 				User.updateDigestSetting(uid, data.dailyDigestFreq, next);
 			},
 			function (next) {
-				cache.del('user:' + uid + ':settings');
-				pubsub.publish('user:settings:cache:del', uid);
 				User.getSettings(uid, next);
 			},
 		], callback);
@@ -213,7 +171,7 @@ module.exports = function (User) {
 		if (!parseInt(uid, 10)) {
 			return setImmediate(callback);
 		}
-		cache.del('user:' + uid + ':settings');
+
 		db.setObjectField('user:' + uid + ':settings', key, value, callback);
 	};
 };

src/views/admin/advanced/cache.tpl

@@ -27,20 +27,21 @@
 		</div>
 
 		<div class="panel panel-default">
-			<div class="panel-heading"><i class="fa fa-calendar-o"></i> User Settings Cache</div>
+			<div class="panel-heading"><i class="fa fa-calendar-o"></i> Object Cache</div>
 			<div class="panel-body">
 
-				<label>[[admin/advanced/cache:items-in-cache]]</label><br/>
-				<span>{userSettingsCache.itemCount}</span><br/>
-
 				<label>[[admin/advanced/cache:length-to-max]]</label><br/>
-				<span>{userSettingsCache.length} / {userSettingsCache.max}</span><br/>
+				<span>{objectCache.length} / {objectCache.max}</span><br/>
 
 				<div class="progress">
-					<div class="progress-bar" role="progressbar" aria-valuenow="{userSettingsCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {userSettingsCache.percentFull}%;">
-						[[admin/advanced/cache:percent-full, {userSettingsCache.percentFull}]]
+					<div class="progress-bar" role="progressbar" aria-valuenow="{objectCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {objectCache.percentFull}%;">
+						[[admin/advanced/cache:percent-full, {objectCache.percentFull}]]
 					</div>
 				</div>
+
+				<!-- IF objectCache.dump -->
+				<pre>{objectCache.dump}</pre>
+				<!-- ENDIF objectCache.dump -->
 			</div>
 		</div>
 
@@ -48,9 +49,6 @@
 			<div class="panel-heading"><i class="fa fa-calendar-o"></i> Group Cache</div>
 			<div class="panel-body">
 
-				<label>[[admin/advanced/cache:items-in-cache]]</label><br/>
-				<span>{groupCache.itemCount}</span><br/>
-
 				<label>[[admin/advanced/cache:length-to-max]]</label><br/>
 				<span>{groupCache.length} / {groupCache.max}</span><br/>
 

test/database/hash.js

@@ -19,8 +19,8 @@ describe('Hash methods', function () {
 	describe('setObject()', function () {
 		it('should create a object', function (done) {
 			db.setObject('testObject1', { foo: 'baris', bar: 99 }, function (err) {
-				assert.equal(err, null);
-				assert.equal(arguments.length, 1);
+				assert.ifError(err);
+				assert(arguments.length < 2);
 				done();
 			});
 		});
@@ -57,16 +57,16 @@ describe('Hash methods', function () {
 	describe('setObjectField()', function () {
 		it('should create a new object with field', function (done) {
 			db.setObjectField('testObject2', 'name', 'ginger', function (err) {
-				assert.equal(err, null);
-				assert.equal(arguments.length, 1);
+				assert.ifError(err);
+				assert(arguments.length < 2);
 				done();
 			});
 		});
 
 		it('should add a new field to an object', function (done) {
 			db.setObjectField('testObject2', 'type', 'cat', function (err) {
-				assert.equal(err, null);
-				assert.equal(arguments.length, 1);
+				assert.ifError(err, null);
+				assert(arguments.length < 2);
 				done();
 			});
 		});
@@ -305,10 +305,10 @@ describe('Hash methods', function () {
 
 		it('should delete an objects field', function (done) {
 			db.deleteObjectField('testObject10', 'delete', function (err) {
-				assert.equal(err, null);
-				assert.equal(arguments.length, 1);
+				assert.ifError(err);
+				assert(arguments.length < 2);
 				db.isObjectField('testObject10', 'delete', function (err, isField) {
-					assert.equal(err, null);
+					assert.ifError(err);
 					assert.equal(isField, false);
 					done();
 				});
@@ -318,7 +318,7 @@ describe('Hash methods', function () {
 		it('should delete multiple fields of the object', function (done) {
 			db.deleteObjectFields('testObject10', ['delete1', 'delete2'], function (err) {
 				assert.ifError(err);
-				assert.equal(arguments.length, 1);
+				assert(arguments.length < 2);
 				async.parallel({
 					delete1: async.apply(db.isObjectField, 'testObject10', 'delete1'),
 					delete2: async.apply(db.isObjectField, 'testObject10', 'delete2'),

test/database/keys.js

@@ -12,15 +12,15 @@ describe('Key methods', function () {
 
 	it('should set a key without error', function (done) {
 		db.set('testKey', 'testValue', function (err) {
-			assert.equal(err, null);
-			assert.equal(arguments.length, 1);
+			assert.ifError(err);
+			assert(arguments.length < 2);
 			done();
 		});
 	});
 
 	it('should get a key without error', function (done) {
 		db.get('testKey', function (err, value) {
-			assert.equal(err, null);
+			assert.ifError(err);
 			assert.equal(arguments.length, 2);
 			assert.strictEqual(value, 'testValue');
 			done();
@@ -29,7 +29,7 @@ describe('Key methods', function () {
 
 	it('should return true if key exist', function (done) {
 		db.exists('testKey', function (err, exists) {
-			assert.equal(err, null);
+			assert.ifError(err);
 			assert.equal(arguments.length, 2);
 			assert.strictEqual(exists, true);
 			done();
@@ -38,7 +38,7 @@ describe('Key methods', function () {
 
 	it('should return false if key does not exist', function (done) {
 		db.exists('doesnotexist', function (err, exists) {
-			assert.equal(err, null);
+			assert.ifError(err);
 			assert.equal(arguments.length, 2);
 			assert.strictEqual(exists, false);
 			done();
@@ -47,11 +47,11 @@ describe('Key methods', function () {
 
 	it('should delete a key without error', function (done) {
 		db.delete('testKey', function (err) {
-			assert.equal(err, null);
-			assert.equal(arguments.length, 1);
+			assert.ifError(err);
+			assert(arguments.length < 2);
 
 			db.get('testKey', function (err, value) {
-				assert.equal(err, null);
+				assert.ifError(err);
 				assert.equal(false, !!value);
 				done();
 			});
@@ -60,10 +60,10 @@ describe('Key methods', function () {
 
 	it('should return false if key was deleted', function (done) {
 		db.delete('testKey', function (err) {
-			assert.equal(err, null);
-			assert.equal(arguments.length, 1);
+			assert.ifError(err);
+			assert(arguments.length < 2);
 			db.exists('testKey', function (err, exists) {
-				assert.equal(err, null);
+				assert.ifError(err);
 				assert.strictEqual(exists, false);
 				done();
 			});
@@ -83,7 +83,7 @@ describe('Key methods', function () {
 				return done(err);
 			}
 			db.deleteAll(['key1', 'key2'], function (err) {
-				assert.equal(err, null);
+				assert.ifError(err);
 				assert.equal(arguments.length, 1);
 				async.parallel({
 					key1exists: function (next) {
@@ -93,7 +93,7 @@ describe('Key methods', function () {
 						db.exists('key2', next);
 					},
 				}, function (err, results) {
-					assert.equal(err, null);
+					assert.ifError(err);
 					assert.equal(results.key1exists, false);
 					assert.equal(results.key2exists, false);
 					done();
@@ -124,7 +124,7 @@ describe('Key methods', function () {
 						db.isSortedSetMember('deletezset', 'value2', next);
 					},
 				}, function (err, results) {
-					assert.equal(err, null);
+					assert.ifError(err);
 					assert.equal(results.key1exists, false);
 					assert.equal(results.key2exists, false);
 					done();
@@ -136,7 +136,7 @@ describe('Key methods', function () {
 	describe('increment', function () {
 		it('should initialize key to 1', function (done) {
 			db.increment('keyToIncrement', function (err, value) {
-				assert.equal(err, null);
+				assert.ifError(err);
 				assert.strictEqual(parseInt(value, 10), 1);
 				done();
 			});
@@ -144,7 +144,7 @@ describe('Key methods', function () {
 
 		it('should increment key to 2', function (done) {
 			db.increment('keyToIncrement', function (err, value) {
-				assert.equal(err, null);
+				assert.ifError(err);
 				assert.strictEqual(parseInt(value, 10), 2);
 				done();
 			});
@@ -158,11 +158,11 @@ describe('Key methods', function () {
 					return done(err);
 				}
 				db.rename('keyOldName', 'keyNewName', function (err) {
-					assert.equal(err, null);
-					assert.equal(arguments.length, 1);
+					assert.ifError(err);
+					assert(arguments.length < 2);
 
 					db.get('keyNewName', function (err, value) {
-						assert.equal(err, null);
+						assert.ifError(err);
 						assert.equal(value, 'renamedKeyValue');
 						done();
 					});