From c379753341a9b937fa06a95f844e95d1d61b3862 Mon Sep 17 00:00:00 2001
From: barisusakli <barisusakli@gmail.com>
Date: Wed, 21 Jan 2015 22:48:22 -0500
Subject: [PATCH] upgrade scripts

upgrade groups to sorted set(groups:createtime) upgrade
upgrade groups:<name>:members to sorted set
new database methods
tests
---
 src/database/mongo/sorted.js | 44 ++++++++++++++++++++++
 src/database/redis/sorted.js | 26 +++++++++++++
 src/groups.js                | 71 +++++++++++++++++++++---------------
 src/upgrade.js               | 51 +++++++++++++++++++++++++-
 tests/database/sorted.js     | 26 +++++++++++++
 5 files changed, 188 insertions(+), 30 deletions(-)

diff --git a/src/database/mongo/sorted.js b/src/database/mongo/sorted.js
index 74da932a18..565189dd8f 100644
--- a/src/database/mongo/sorted.js
+++ b/src/database/mongo/sorted.js
@@ -400,6 +400,50 @@ module.exports = function(db, module) {
 		});
 	};
 
+	module.isMemberOfSortedSets = function(keys, value, callback) {
+		if (!Array.isArray(keys)) {
+			return callback();
+		}
+		value = helpers.valueToString(value);
+		db.collection('objects').find({_key: {$in: keys}, value: value}, {fields: {_id: 0, _key: 1, value: 1}}).toArray(function(err, results) {
+			if (err) {
+				return callback(err);
+			}
+
+			results = results.map(function(item) {
+				return item._key;
+			});
+
+			results = keys.map(function(key) {
+				return results.indexOf(key) !== -1;
+			});
+			callback(null, results);
+		});
+	};
+
+	module.getSortedSetsMembers = function(keys, callback) {
+		if (!Array.isArray(keys) || !keys.length) {
+			return callback(null, []);
+		}
+		db.collection('objects').find({_key: {$in: keys}}, {_id: 0, _key: 1, value: 1}).toArray(function(err, data) {
+			if (err) {
+				return callback(err);
+			}
+
+			var sets = {};
+			data.forEach(function(set) {
+			 	sets[set._key] = sets[set._key] || [];
+			 	sets[set._key].push(set.value);
+			});
+
+			var returnData = new Array(keys.length);
+			for(var i=0; i<keys.length; ++i) {
+			 	returnData[i] = sets[keys[i]] || [];
+			}
+			callback(null, returnData);
+		});
+	};
+
 	module.getSortedSetUnion = function(sets, start, stop, callback) {
 		getSortedSetUnion(sets, 1, start, stop, callback);
 	};
diff --git a/src/database/redis/sorted.js b/src/database/redis/sorted.js
index cbee81c997..576554d458 100644
--- a/src/database/redis/sorted.js
+++ b/src/database/redis/sorted.js
@@ -214,6 +214,32 @@ module.exports = function(redisClient, module) {
 		});
 	};
 
+	module.isMemberOfSortedSets = function(keys, value, callback) {
+		var multi = redisClient.multi();
+		for (var i=0; i<keys.length; ++i) {
+			multi.zscore(keys[i], value);
+		}
+		multi.exec(function(err, results) {
+			if (err) {
+				return callback(err);
+			}
+			results = results.map(function(score) {
+				return !!score;
+			});
+			callback(null, results);
+		});
+	};
+
+	module.getSortedSetsMembers = function(keys, callback) {
+		var multi = redisClient.multi();
+		for (var i=0; i<keys.length; ++i) {
+			multi.zrange(keys[i], 0, -1);
+		}
+		multi.exec(function(err, results) {
+			callback(err, results);
+		});
+	};
+
 	function multi(command, keys, value, callback) {
 		var	m = redisClient.multi();
 
diff --git a/src/groups.js b/src/groups.js
index 2c510fc119..55d40e3559 100644
--- a/src/groups.js
+++ b/src/groups.js
@@ -76,7 +76,7 @@ var async = require('async'),
 		};
 
 	Groups.list = function(options, callback) {
-		db.getSetMembers('groups', function (err, groupNames) {
+		db.getSortedSetRevRange('groups:createtime', 0, -1, function (err, groupNames) {
 			if (err) {
 				return callback(err);
 			}
@@ -103,7 +103,7 @@ var async = require('async'),
 				}
 			},
 			users: function (next) {
-				db.getSetMembers('group:' + groupName + ':members', function (err, uids) {
+				db.getSortedSetRevRange('group:' + groupName + ':members', 0, -1, function (err, uids) {
 					if (err) {
 						return next(err);
 					}
@@ -208,6 +208,7 @@ var async = require('async'),
 			results.base.name = validator.escape(results.base.name);
 			results.base.description = validator.escape(results.base.description);
 			results.base.userTitle = validator.escape(results.base.userTitle);
+			results.base.createtime = utils.toISOString(results.base.createtime);
 			results.base.members = results.users.filter(Boolean);
 			results.base.pending = results.pending.filter(Boolean);
 			results.base.count = numUsers || results.base.members.length;
@@ -222,6 +223,7 @@ var async = require('async'),
 			results.base.isPending = results.isPending;
 			results.base.isOwner = results.isOwner;
 
+
 			plugins.fireHook('filter:group.get', {group: results.base}, function(err, data) {
 				callback(err, data ? data.group : null);
 			});
@@ -262,18 +264,18 @@ var async = require('async'),
 	};
 
 	Groups.getMembers = function(groupName, callback) {
-		db.getSetMembers('group:' + groupName + ':members', callback);
+		db.getSortedSetRevRange('group:' + groupName + ':members', 0, -1, callback);
 	};
 
 	Groups.isMember = function(uid, groupName, callback) {
 		if (!uid || parseInt(uid, 10) <= 0) {
 			return callback(null, false);
 		}
-		db.isSetMember('group:' + groupName + ':members', uid, callback);
+		db.isSortedSetMember('group:' + groupName + ':members', uid, callback);
 	};
 
 	Groups.isMembers = function(uids, groupName, callback) {
-		db.isSetMembers('group:' + groupName + ':members', uids, callback);
+		db.isSortedSetMembers('group:' + groupName + ':members', uids, callback);
 	};
 
 	Groups.isMemberOfGroups = function(uid, groups, callback) {
@@ -283,15 +285,16 @@ var async = require('async'),
 		groups = groups.map(function(groupName) {
 			return 'group:' + groupName + ':members';
 		});
-		db.isMemberOfSets(groups, uid, callback);
+
+		db.isMemberOfSortedSets(groups, uid, callback);
 	};
 
 	Groups.getMemberCount = function(groupName, callback) {
-		db.setCount('group:' + groupName + ':members', callback);
+		db.sortedSetCard('group:' + groupName + ':members', callback);
 	};
 
 	Groups.isMemberOfGroupList = function(uid, groupListKey, callback) {
-		db.getSetMembers('group:' + groupListKey + ':members', function(err, groupNames) {
+		db.getSortedSetRange('group:' + groupListKey + ':members', 0, -1, function(err, groupNames) {
 			if (err) {
 				return callback(err);
 			}
@@ -315,7 +318,7 @@ var async = require('async'),
 			return 'group:' + groupName + ':members';
 		});
 
-		db.getSetsMembers(sets, function(err, members) {
+		db.getSortedSetsMembers(sets, function(err, members) {
 			if (err) {
 				return callback(err);
 			}
@@ -349,7 +352,7 @@ var async = require('async'),
 	};
 
 	Groups.isMembersOfGroupList = function(uids, groupListKey, callback) {
-		db.getSetMembers('group:' + groupListKey + ':members', function(err, groupNames) {
+		db.getSortedSetRange('group:' + groupListKey + ':members', 0, -1, function(err, groupNames) {
 			if (err) {
 				return callback(err);
 			}
@@ -389,7 +392,7 @@ var async = require('async'),
 				});
 			async.parallel([
 				async.apply(db.isObjectFields, 'groupslug:groupname', slugs),
-				async.apply(db.isSetMembers, 'groups', name)
+				async.apply(db.isSortedSetMember, 'groups:createtime', name)
 			], function(err, results) {
 				if (err) {
 					return callback(err);
@@ -403,7 +406,7 @@ var async = require('async'),
 			var slug = utils.slugify(name);
 			async.parallel([
 				async.apply(db.isObjectField, 'groupslug:groupname', slug),
-				async.apply(db.isSetMember, 'groups', name)
+				async.apply(db.isSortedSetMember, 'groups:createtime', name)
 			], function(err, results) {
 				callback(err, !err ? (results[0] || results[1]) : null);
 			});
@@ -427,11 +430,13 @@ var async = require('async'),
 			if (exists) {
 				return callback(new Error('[[error:group-already-exists]]'));
 			}
+			var now = Date.now();
 
 			var slug = utils.slugify(data.name),
 				groupData = {
 					name: data.name,
 					slug: slug,
+					createtime: now,
 					userTitle: data.name,
 					description: data.description || '',
 					deleted: '0',
@@ -440,13 +445,13 @@ var async = require('async'),
 					'private': data.private || '1'
 				},
 				tasks = [
-					async.apply(db.setAdd, 'groups', data.name),
+					async.apply(db.sortedSetAdd, 'groups:createtime', now, data.name),
 					async.apply(db.setObject, 'group:' + data.name, groupData)
 				];
 
 			if (data.hasOwnProperty('ownerUid')) {
 				tasks.push(async.apply(db.setAdd, 'group:' + data.name + ':owners', data.ownerUid));
-				tasks.push(async.apply(db.setAdd, 'group:' + data.name + ':members', data.ownerUid));
+				tasks.push(async.apply(db.sortedSetAdd, 'group:' + data.name + ':members', now, data.ownerUid));
 			}
 
 			if (!data.hidden) {
@@ -531,7 +536,7 @@ var async = require('async'),
 						db.setObjectField('groupslug:groupname', utils.slugify(newName), newName, next);
 					},
 					function(next) {
-						db.getSetMembers('groups', function(err, groups) {
+						db.getSortedSetRange('groups:createtime', 0, -1, function(err, groups) {
 							if (err) {
 								return next(err);
 							}
@@ -556,7 +561,7 @@ var async = require('async'),
 						});
 					},
 					function(next) {
-						renameGroupMember('groups', oldName, newName, next);
+						renameGroupMember('groups:createtime', oldName, newName, next);
 					},
 					function(next) {
 						plugins.fireHook('action:group.rename', {
@@ -572,16 +577,21 @@ var async = require('async'),
 	}
 
 	function renameGroupMember(group, oldName, newName, callback) {
-		db.isSetMember(group, oldName, function(err, isMember) {
+		db.isSortedSetMember(group, oldName, function(err, isMember) {
 			if (err || !isMember) {
 				return callback(err);
 			}
-			async.series([
+			var score;
+			async.waterfall([
 				function (next) {
-					db.setRemove(group, oldName, next);
+					db.sortedSetScore(group, oldName, next);
+				},
+				function (_score, next) {
+					score = _score;
+					db.sortedSetRemove(group, oldName, next);
 				},
 				function (next) {
-					db.setAdd(group, newName, next);
+					db.sortedSetAdd(group, score, newName, next);
 				}
 			], callback);
 		});
@@ -593,18 +603,18 @@ var async = require('async'),
 
 			async.parallel([
 				async.apply(db.delete, 'group:' + groupName),
-				async.apply(db.setRemove, 'groups', groupName),
+				async.apply(db.sortedSetRemove, 'groups:createtime', groupName),
 				async.apply(db.delete, 'group:' + groupName + ':members'),
 				async.apply(db.delete, 'group:' + groupName + ':pending'),
 				async.apply(db.delete, 'group:' + groupName + ':owners'),
 				async.apply(db.deleteObjectField, 'groupslug:groupname', utils.slugify(groupName)),
 				function(next) {
-					db.getSetMembers('groups', function(err, groups) {
+					db.getSortedSetRange('groups:createtime', 0, -1, function(err, groups) {
 						if (err) {
 							return next(err);
 						}
 						async.each(groups, function(group, next) {
-							db.setRemove('group:' + group + ':members', groupName, next);
+							db.sortedSetRemove('group:' + group + ':members', groupName, next);
 						}, next);
 					});
 				}
@@ -617,7 +627,7 @@ var async = require('async'),
 
 		Groups.exists(groupName, function(err, exists) {
 			if (exists) {
-				db.setAdd('group:' + groupName + ':members', uid, callback);
+				db.sortedSetAdd('group:' + groupName + ':members', Date.now(), uid, callback);
 				plugins.fireHook('action:group.join', {
 					groupName: groupName,
 					uid: uid
@@ -633,7 +643,7 @@ var async = require('async'),
 						return callback(err);
 					}
 
-					db.setAdd('group:' + groupName + ':members', uid, callback);
+					db.sortedSetAdd('group:' + groupName + ':members', Date.now(), uid, callback);
 					plugins.fireHook('action:group.join', {
 						groupName: groupName,
 						uid: uid
@@ -680,7 +690,7 @@ var async = require('async'),
 	Groups.leave = function(groupName, uid, callback) {
 		callback = callback || function() {};
 
-		db.setRemove('group:' + groupName + ':members', uid, function(err) {
+		db.sortedSetRemove('group:' + groupName + ':members', uid, function(err) {
 			if (err) {
 				return callback(err);
 			}
@@ -706,7 +716,10 @@ var async = require('async'),
 	};
 
 	Groups.leaveAllGroups = function(uid, callback) {
-		db.getSetMembers('groups', function(err, groups) {
+		db.getSortedSetRange('groups:createtime', 0, -1, function(err, groups) {
+			if (err) {
+				return callback(err);
+			}
 			async.each(groups, function(groupName, next) {
 				Groups.isMember(uid, groupName, function(err, isMember) {
 					if (!err && isMember) {
@@ -743,7 +756,7 @@ var async = require('async'),
 	};
 
 	Groups.getUserGroups = function(uids, callback) {
-		db.getSetMembers('groups', function(err, groupNames) {
+		db.getSortedSetRevRange('groups:createtime', 0, -1, function(err, groupNames) {
 			if (err) {
 				return callback(err);
 			}
@@ -775,7 +788,7 @@ var async = require('async'),
 				});
 
 				async.map(uids, function(uid, next) {
-					db.isMemberOfSets(groupSets, uid, function(err, isMembers) {
+					db.isMemberOfSortedSets(groupSets, uid, function(err, isMembers) {
 						if (err) {
 							return next(err);
 						}
diff --git a/src/upgrade.js b/src/upgrade.js
index b957adbe7d..506639c463 100644
--- a/src/upgrade.js
+++ b/src/upgrade.js
@@ -21,7 +21,7 @@ var db = require('./database'),
 	schemaDate, thisSchemaDate,
 
 	// IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema
-	latestSchema = Date.UTC(2015, 0, 19);
+	latestSchema = Date.UTC(2015, 0, 21);
 
 Upgrade.check = function(callback) {
 	db.get('schemaDate', function(err, value) {
@@ -713,6 +713,55 @@ Upgrade.upgrade = function(callback) {
 				winston.info('[2015/01/19] Generating group slugs skipped');
 				next();
 			}
+		},
+		function(next) {
+			thisSchemaDate = Date.UTC(2015, 0, 21);
+			if (schemaDate < thisSchemaDate) {
+				updatesMade = true;
+				winston.info('[2015/01/21] Upgrading groups to sorted set');
+
+				db.getSetMembers('groups', function(err, groupNames) {
+					if (err) {
+						return next(err);
+					}
+
+					var now = Date.now();
+					async.each(groupNames, function(groupName, next) {
+						db.getSetMembers('group:' + groupName + ':members', function(err, members) {
+							if (err) {
+								return next(err);
+							}
+
+							async.series([
+								function(next) {
+									if (members && members.length) {
+										db.delete('group:' + groupName + ':members', function(err) {
+											if (err) {
+												return next(err);
+											}
+											var scores = members.map(function() {
+												return now;
+											});
+											db.sortedSetAdd('group:' + groupName + ':members', scores, members, next);
+										});
+									} else {
+										next();
+									}
+								},
+								async.apply(db.sortedSetAdd, 'groups:createtime', now, groupName),
+								async.apply(db.setObjectField, 'group:' + groupName, 'createtime', now)
+							], next);
+						});
+
+					}, function(err) {
+						winston.info('[2015/01/21] Upgrading groups to sorted set done');
+						Upgrade.update(thisSchemaDate, next);
+					});
+				});
+			} else {
+				winston.info('[2015/01/21] Upgrading groups to sorted set skipped');
+				next();
+			}
 		}
 
 		// Add new schema updates here
diff --git a/tests/database/sorted.js b/tests/database/sorted.js
index 1739989689..ed0bea521f 100644
--- a/tests/database/sorted.js
+++ b/tests/database/sorted.js
@@ -377,6 +377,32 @@ describe('Sorted Set methods', function() {
 		});
 	});
 
+	describe('isMemberOfSortedSets', function() {
+		it('should return true for members false for non members', function(done) {
+			db.isMemberOfSortedSets(['doesnotexist', 'sortedSetTest1', 'sortedSetTest2'], 'value2', function(err, isMembers) {
+				assert.equal(err, null);
+				assert.equal(arguments.length, 2);
+				assert.deepEqual(isMembers, [false, true, false]);
+				done();
+			});
+		});
+	});
+
+	describe('getSortedSetsMembers', function() {
+		it('should return members of multiple sorted sets', function(done) {
+			db.getSortedSetMembers(['doesnotexist', 'sortedSetTest1'], function(err, sortedSets) {
+				assert.equal(err, null);
+				assert.equal(arguments.length, 2);
+				assert.deepEqual(sortedSets[0], []);
+				sortedSets[0].forEach(function(element) {
+					assert.notEqual(['value1', 'value2', 'value3'].indexOf(element), -1);
+				});
+
+				done();
+			});
+		});
+	});
+
 	describe('getSortedSetUnion()', function() {
 		it('should return an array of values from both sorted sets sorted by scores lowest to highest', function(done) {
 			db.getSortedSetUnion(['sortedSetTest2', 'sortedSetTest3'], 0, -1, function(err, values) {