From c56b30ff6044eea8919a924fec2faeea96ffc432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Thu, 7 May 2015 13:43:06 -0400 Subject: [PATCH] convert uid mappings to sorted sets email:uid, username:uid, userslug:uid, fullname:uid all converted to sorted sets prevents hitting mongodb document size limit --- src/upgrade.js | 46 +++++++++++++++++++++++++++++++-- src/user.js | 18 ++++--------- src/user/admin.js | 2 +- src/user/create.js | 6 ++--- src/user/delete.js | 8 +++--- src/user/email.js | 2 +- src/user/profile.js | 63 ++++++++++++++++++--------------------------- src/user/search.js | 12 +++++---- tests/user.js | 28 ++++++++++++++++++++ 9 files changed, 118 insertions(+), 67 deletions(-) diff --git a/src/upgrade.js b/src/upgrade.js index 5b7c8f65ed..3b9492b94b 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -21,7 +21,7 @@ var db = require('./database'), schemaDate, thisSchemaDate, // IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema - latestSchema = Date.UTC(2015, 1, 25, 6); + latestSchema = Date.UTC(2015, 4, 7); Upgrade.check = function(callback) { db.get('schemaDate', function(err, value) { @@ -938,7 +938,7 @@ Upgrade.upgrade = function(callback) { } winston.info('[2015/02/24] Upgrading privilege groups to system groups done'); Upgrade.update(thisSchemaDate, next); - }) + }); }); } else { winston.info('[2015/02/24] Upgrading privilege groups to system groups skipped'); @@ -963,6 +963,48 @@ Upgrade.upgrade = function(callback) { winston.info('[2015/02/25] Upgrading menu items to dynamic navigation system skipped'); next(); } + }, + function(next) { + function upgradeHashToSortedSet(hash, callback) { + db.getObject(hash, function(err, oldHash) { + if (err) { + return callback(err); + } + db.rename(hash, hash + '_old', function(err) { + if (err) { + return callback(err); + } + var keys = Object.keys(oldHash); + async.each(keys, function(key, next) { + db.sortedSetAdd(hash, oldHash[key], key, next); + }, callback); + }); + }); + } + + thisSchemaDate = Date.UTC(2015, 4, 7); + if (schemaDate < thisSchemaDate) { + updatesMade = true; + winston.info('[2015/02/25] Upgrading uid mappings to sorted set'); + + async.series([ + async.apply(upgradeHashToSortedSet, 'email:uid'), + async.apply(upgradeHashToSortedSet, 'fullname:uid'), + async.apply(upgradeHashToSortedSet, 'username:uid'), + async.apply(upgradeHashToSortedSet, 'userslug:uid'), + ], function(err) { + if (err) { + return next(err); + } + + winston.info('[2015/05/07] Upgrading uid mappings to sorted set done'); + Upgrade.update(thisSchemaDate, next); + }); + + } else { + winston.info('[2015/05/07] Upgrading uid mappings to sorted set skipped'); + next(); + } } // Add new schema updates here diff --git a/src/user.js b/src/user.js index 7ad9810554..ff8f6fbb36 100644 --- a/src/user.js +++ b/src/user.js @@ -334,26 +334,18 @@ var async = require('async'), if (!username) { return callback(); } - db.getObjectField('username:uid', username, callback); + db.sortedSetScore('username:uid', username, callback); }; User.getUidsByUsernames = function(usernames, callback) { - db.getObjectFields('username:uid', usernames, function(err, users) { - if (err) { - return callback(err); - } - var uids = usernames.map(function(username) { - return users[username]; - }); - callback(null, uids); - }); + db.sortedSetScores('username:uid', usernames, callback); }; User.getUidByUserslug = function(userslug, callback) { if (!userslug) { return callback(); } - db.getObjectField('userslug:uid', userslug, callback); + db.sortedSetScore('userslug:uid', userslug, callback); }; User.getUsernamesByUids = function(uids, callback) { @@ -382,11 +374,11 @@ var async = require('async'), }; User.getUidByEmail = function(email, callback) { - db.getObjectField('email:uid', email.toLowerCase(), callback); + db.sortedSetScore('email:uid', email.toLowerCase(), callback); }; User.getUsernameByEmail = function(email, callback) { - db.getObjectField('email:uid', email.toLowerCase(), function(err, uid) { + db.sortedSetScore('email:uid', email.toLowerCase(), function(err, uid) { if (err) { return callback(err); } diff --git a/src/user/admin.js b/src/user/admin.js index 4d559fcb34..615a765f6a 100644 --- a/src/user/admin.js +++ b/src/user/admin.js @@ -32,7 +32,7 @@ module.exports = function(User) { async.waterfall([ function(next) { - db.getObjectValues('username:uid', next); + db.getSortedSetRange('username:uid', 0, -1, next); }, function(uids, next) { User.getMultipleUserFields(uids, ['uid', 'email', 'username'], next); diff --git a/src/user/create.js b/src/user/create.js index 4349b8db53..d4a3f51adb 100644 --- a/src/user/create.js +++ b/src/user/create.js @@ -91,10 +91,10 @@ module.exports = function(User) { function(next) { async.parallel([ function(next) { - db.setObjectField('username:uid', userData.username, userData.uid, next); + db.sortedSetAdd('username:uid', userData.uid, userData.username, next); }, function(next) { - db.setObjectField('userslug:uid', userData.userslug, userData.uid, next); + db.sortedSetAdd('userslug:uid', userData.uid, userData.userslug, next); }, function(next) { db.sortedSetAdd('users:joindate', timestamp, userData.uid, next); @@ -107,7 +107,7 @@ module.exports = function(User) { }, function(next) { if (userData.email) { - db.setObjectField('email:uid', userData.email.toLowerCase(), userData.uid, next); + db.sortedSetAdd('email:uid', userData.uid, userData.email.toLowerCase(), next); if (parseInt(userData.uid, 10) !== 1 && parseInt(meta.config.requireEmailConfirmation, 10) === 1) { User.email.sendValidationEmail(userData.uid, userData.email); } diff --git a/src/user/delete.js b/src/user/delete.js index 49e21553c9..2d06caf745 100644 --- a/src/user/delete.js +++ b/src/user/delete.js @@ -49,17 +49,17 @@ module.exports = function(User) { async.parallel([ function(next) { - db.deleteObjectField('username:uid', userData.username, next); + db.sortedSetRemove('username:uid', userData.username, next); }, function(next) { - db.deleteObjectField('userslug:uid', userData.userslug, next); + db.sortedSetRemove('userslug:uid', userData.userslug, next); }, function(next) { - db.deleteObjectField('fullname:uid', userData.fullname, next); + db.sortedSetRemove('fullname:uid', userData.fullname, next); }, function(next) { if (userData.email) { - db.deleteObjectField('email:uid', userData.email.toLowerCase(), next); + db.sortedSetRemove('email:uid', userData.email.toLowerCase(), next); } else { next(); } diff --git a/src/user/email.js b/src/user/email.js index b4c6460683..c5beb6c1e3 100644 --- a/src/user/email.js +++ b/src/user/email.js @@ -22,7 +22,7 @@ var async = require('async'), }; UserEmail.available = function(email, callback) { - db.isObjectField('email:uid', email.toLowerCase(), function(err, exists) { + db.isSortedSetMember('email:uid', email.toLowerCase(), function(err, exists) { callback(err, !exists); }); }; diff --git a/src/user/profile.js b/src/user/profile.js index 7265d5896b..d10aba66db 100644 --- a/src/user/profile.js +++ b/src/user/profile.js @@ -165,7 +165,7 @@ module.exports = function(User) { return callback(); } - db.deleteObjectField('email:uid', userData.email.toLowerCase(), function(err) { + db.sortedSetRemove('email:uid', userData.email.toLowerCase(), function(err) { if (err) { return callback(err); } @@ -176,7 +176,7 @@ module.exports = function(User) { User.setUserField(uid, 'gravatarpicture', gravatarpicture, next); }, function(next) { - db.setObjectField('email:uid', newEmail.toLowerCase(), uid, next); + db.sortedSetAdd('email:uid', uid, newEmail.toLowerCase(), next); }, function(next) { User.setUserField(uid, 'email', newEmail, next); @@ -205,61 +205,37 @@ module.exports = function(User) { } User.getUserFields(uid, ['username', 'userslug'], function(err, userData) { - function update(field, object, value, callback) { - async.series([ - function(next) { - db.deleteObjectField(field + ':uid', userData[field], next); - }, - function(next) { - User.setUserField(uid, field, value, next); - }, - function(next) { - db.setObjectField(object, value, uid, next); - } - ], callback); - } - if (err) { return callback(err); } async.parallel([ function(next) { - if (newUsername === userData.username) { - return next(); - } - - update('username', 'username:uid', newUsername, next); + updateUidMapping('username', uid, newUsername, userData.username, next); }, function(next) { var newUserslug = utils.slugify(newUsername); - if (newUserslug === userData.userslug) { - return next(); - } - - update('userslug', 'userslug:uid', newUserslug, next); + updateUidMapping('userslug', uid, newUserslug, userData.userslug, next); } ], callback); }); } - function updateFullname(uid, newFullname, callback) { - async.waterfall([ + function updateUidMapping(field, uid, value, oldValue, callback) { + if (value === oldValue) { + return callback(); + } + + async.series([ function(next) { - User.getUserField(uid, 'fullname', next); - }, - function(fullname, next) { - if (newFullname === fullname) { - return callback(); - } - db.deleteObjectField('fullname:uid', fullname, next); + db.sortedSetRemove(field + ':uid', oldValue, next); }, function(next) { - User.setUserField(uid, 'fullname', newFullname, next); + User.setUserField(uid, field, value, next); }, function(next) { - if (newFullname) { - db.setObjectField('fullname:uid', newFullname, uid, next); + if (value) { + db.sortedSetAdd(field + ':uid', uid, value, next); } else { next(); } @@ -267,6 +243,17 @@ module.exports = function(User) { ], callback); } + function updateFullname(uid, newFullname, callback) { + async.waterfall([ + function(next) { + User.getUserField(uid, 'fullname', next); + }, + function(fullname, next) { + updateUidMapping('fullname', uid, newFullname, fullname, next); + } + ], callback); + } + User.changePassword = function(uid, data, callback) { if (!uid || !data || !data.uid) { return callback(new Error('[[error:invalid-uid]]')); diff --git a/src/user/search.js b/src/user/search.js index 5a0d60086c..ce1ad0c6b2 100644 --- a/src/user/search.js +++ b/src/user/search.js @@ -84,7 +84,9 @@ module.exports = function(User) { return searchBy + ':uid'; }); - db.getObjects(keys, function(err, hashes) { + async.map(keys, function(key, next) { + db.getSortedSetRangeWithScores(key, 0, -1, next); + }, function(err, hashes) { if (err || !hashes) { return callback(err, []); } @@ -97,16 +99,16 @@ module.exports = function(User) { var resultsPerPage = parseInt(meta.config.userSearchResultsPerPage, 10) || 20; var hardCap = resultsPerPage * 10; - for(var i=0; i= hardCap) { break; } } } - if (uids.length >= hardCap) { break; } diff --git a/tests/user.js b/tests/user.js index 9a1ee4b90f..cb37c8e27c 100644 --- a/tests/user.js +++ b/tests/user.js @@ -36,6 +36,7 @@ describe('User', function() { beforeEach(function(){ userData = { username: 'John Smith', + fullname: 'John Smith McNamara', password: 'swordfish', email: 'john@example.com', callback: undefined @@ -254,6 +255,33 @@ describe('User', function() { }); }); + describe('hash methods', function() { + + it('should return uid from email', function(next) { + User.getUidByEmail('john@example.com', function(err, uid) { + assert.ifError(err); + assert.equal(parseInt(uid, 10), parseInt(testUid, 10)); + done(); + }); + }); + + it('should return uid from username', function(next) { + User.getUidByUsername('John Smith', function(err, uid) { + assert.ifError(err); + assert.equal(parseInt(uid, 10), parseInt(testUid, 10)); + done(); + }); + }); + + it('should return uid from userslug', function(next) { + User.getUidByUserslug('john-smith', function(err, uid) { + assert.ifError(err); + assert.equal(parseInt(uid, 10), parseInt(testUid, 10)); + done(); + }); + }); + }); + after(function() { db.flushdb(); });