update groups join to take array of group names (#6834)

* allow groups.join to take an array of group names

* pass an array to groups.join/leave in privileges

* split up groups/membership

* add hits/miss to group cache

* fix typo
v1.18.x
Barış Soner Uşaklı 6 years ago committed by GitHub
parent 523d68c640
commit b57db7fd8e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -75,12 +75,9 @@ module.exports = function (Categories) {
}
Categories.parseDescription(category.cid, category.description, next);
},
async.apply(db.sortedSetAdd, 'categories:cid', category.order, category.cid),
async.apply(db.sortedSetAdd, 'cid:' + parentCid + ':children', category.order, category.cid),
async.apply(privileges.categories.give, defaultPrivileges, category.cid, 'administrators'),
async.apply(privileges.categories.give, defaultPrivileges, category.cid, 'registered-users'),
async.apply(privileges.categories.give, ['find', 'read', 'topics:read'], category.cid, 'guests'),
async.apply(privileges.categories.give, ['find', 'read', 'topics:read'], category.cid, 'spiders'),
async.apply(db.sortedSetsAdd, ['categories:cid', 'cid:' + parentCid + ':children'], category.order, category.cid),
async.apply(privileges.categories.give, defaultPrivileges, category.cid, ['administrators', 'registered-users']),
async.apply(privileges.categories.give, ['find', 'read', 'topics:read'], category.cid, ['guests', 'spiders']),
], next);
},
function (results, next) {

@ -30,6 +30,9 @@ cacheController.get = function (req, res) {
itemCount: groupCache.itemCount,
percentFull: ((groupCache.length / groupCache.max) * 100).toFixed(2),
dump: req.query.debug ? JSON.stringify(groupCache.dump(), null, 4) : false,
hits: utils.addCommas(String(groupCache.hits)),
misses: utils.addCommas(String(groupCache.misses)),
hitRatio: (groupCache.hits / (groupCache.hits + groupCache.misses)).toFixed(4),
},
};

@ -20,6 +20,9 @@ require('./groups/search')(Groups);
require('./groups/cover')(Groups);
require('./groups/posts')(Groups);
require('./groups/user')(Groups);
require('./groups/join')(Groups);
require('./groups/leave')(Groups);
require('./groups/cache')(Groups);
Groups.ephemeralGroups = ['guests', 'spiders'];

@ -0,0 +1,48 @@
'use strict';
var pubsub = require('../pubsub');
var LRU = require('lru-cache');
var cache = LRU({
max: 40000,
maxAge: 0,
});
cache.hits = 0;
cache.misses = 0;
module.exports = function (Groups) {
Groups.cache = cache;
pubsub.on('group:cache:reset', function () {
localReset();
});
pubsub.on('group:cache:del', function (data) {
if (data && data.groupNames) {
data.groupNames.forEach(function (groupName) {
cache.del(data.uid + ':' + groupName);
});
}
});
Groups.resetCache = function () {
pubsub.publish('group:cache:reset');
localReset();
};
function localReset() {
cache.reset();
cache.hits = 0;
cache.misses = 0;
}
Groups.clearCache = function (uid, groupNames) {
if (!Array.isArray(groupNames)) {
groupNames = [groupNames];
}
pubsub.publish('group:cache:del', { uid: uid, groupNames: groupNames });
groupNames.forEach(function (groupName) {
cache.del(uid + ':' + groupName);
});
};
};

@ -0,0 +1,124 @@
'use strict';
const async = require('async');
const winston = require('winston');
const db = require('../database');
const user = require('../user');
const plugins = require('../plugins');
module.exports = function (Groups) {
Groups.join = function (groupNames, uid, callback) {
callback = callback || function () {};
if (!groupNames) {
return callback(new Error('[[error:invalid-data]]'));
}
if (!Array.isArray(groupNames)) {
groupNames = [groupNames];
}
if (!uid) {
return callback(new Error('[[error:invalid-uid]]'));
}
var isAdmin;
async.waterfall([
function (next) {
async.parallel({
isMembers: async.apply(Groups.isMemberOfGroups, uid, groupNames),
exists: async.apply(Groups.exists, groupNames),
isAdmin: async.apply(user.isAdministrator, uid),
}, next);
},
function (results, next) {
isAdmin = results.isAdmin;
var groupsToCreate = groupNames.filter(function (groupName, index) {
return groupName && !results.exists[index];
});
groupNames = groupNames.filter(function (groupName, index) {
return !results.isMembers[index];
});
if (!groupNames.length) {
return callback();
}
createNonExistingGroups(groupsToCreate, next);
},
function (next) {
var tasks = [
async.apply(db.sortedSetsAdd, groupNames.map(groupName => 'group:' + groupName + ':members'), Date.now(), uid),
async.apply(db.incrObjectField, groupNames.map(groupName => 'group:' + groupName), 'memberCount'),
];
if (isAdmin) {
tasks.push(async.apply(db.setsAdd, groupNames.map(groupName => 'group:' + groupName + ':owners'), uid));
}
async.parallel(tasks, next);
},
function (results, next) {
Groups.clearCache(uid, groupNames);
Groups.getGroupsFields(groupNames, ['name', 'hidden', 'memberCount'], next);
},
function (groupData, next) {
var visibleGroups = groupData.filter(function (groupData) {
return groupData && parseInt(groupData.hidden, 10) !== 1;
});
if (visibleGroups.length) {
db.sortedSetAdd('groups:visible:memberCount', visibleGroups.map(groupData => groupData.memberCount), visibleGroups.map(groupData => groupData.name), next);
} else {
next();
}
},
function (next) {
setGroupTitleIfNotSet(groupNames, uid, next);
},
function (next) {
plugins.fireHook('action:group.join', {
groupNames: groupNames,
uid: uid,
});
next();
},
], callback);
};
function createNonExistingGroups(groupsToCreate, callback) {
if (!groupsToCreate.length) {
return setImmediate(callback);
}
async.eachSeries(groupsToCreate, function (groupName, next) {
Groups.create({
name: groupName,
hidden: 1,
}, function (err) {
if (err && err.message !== '[[error:group-already-exists]]') {
winston.error('[groups.join] Could not create new hidden group', err);
return next(err);
}
next();
});
}, callback);
}
function setGroupTitleIfNotSet(groupNames, uid, callback) {
groupNames = groupNames.filter(function (groupName) {
return groupName !== 'registered-users' && !Groups.isPrivilegeGroup(groupName);
});
if (!groupNames.length) {
return callback();
}
db.getObjectField('user:' + uid, 'groupTitle', function (err, currentTitle) {
if (err || currentTitle || currentTitle === '') {
return callback(err);
}
user.setUserField(uid, 'groupTitle', JSON.stringify(groupNames), callback);
});
}
};

@ -0,0 +1,122 @@
'use strict';
const async = require('async');
const db = require('../database');
const user = require('../user');
const plugins = require('../plugins');
module.exports = function (Groups) {
Groups.leave = function (groupNames, uid, callback) {
callback = callback || function () {};
if (!Array.isArray(groupNames)) {
groupNames = [groupNames];
}
async.waterfall([
function (next) {
async.parallel({
isMembers: async.apply(Groups.isMemberOfGroups, uid, groupNames),
exists: async.apply(Groups.exists, groupNames),
}, next);
},
function (result, next) {
groupNames = groupNames.filter(function (groupName, index) {
return result.isMembers[index] && result.exists[index];
});
if (!groupNames.length) {
return callback();
}
async.parallel([
async.apply(db.sortedSetRemove, groupNames.map(groupName => 'group:' + groupName + ':members'), uid),
async.apply(db.setRemove, groupNames.map(groupName => 'group:' + groupName + ':owners'), uid),
async.apply(db.decrObjectField, groupNames.map(groupName => 'group:' + groupName), 'memberCount'),
], next);
},
function (results, next) {
Groups.clearCache(uid, groupNames);
Groups.getGroupsFields(groupNames, ['name', 'hidden', 'memberCount'], next);
},
function (groupData, next) {
if (!groupData) {
return callback();
}
var tasks = [];
var emptyPrivilegeGroups = groupData.filter(function (groupData) {
return groupData && Groups.isPrivilegeGroup(groupData.name) && parseInt(groupData.memberCount, 10) === 0;
});
if (emptyPrivilegeGroups.length) {
tasks.push(async.apply(Groups.destroy, emptyPrivilegeGroups));
}
var visibleGroups = groupData.filter(function (groupData) {
return groupData && parseInt(groupData.hidden, 10) !== 1;
});
if (visibleGroups.length) {
tasks.push(async.apply(db.sortedSetAdd, 'groups:visible:memberCount', visibleGroups.map(groupData => groupData.memberCount), visibleGroups.map(groupData => groupData.name)));
}
async.parallel(tasks, function (err) {
next(err);
});
},
function (next) {
clearGroupTitleIfSet(groupNames, uid, next);
},
function (next) {
plugins.fireHook('action:group.leave', {
groupNames: groupNames,
uid: uid,
});
next();
},
], callback);
};
function clearGroupTitleIfSet(groupNames, uid, callback) {
groupNames = groupNames.filter(function (groupName) {
return groupName !== 'registered-users' && !Groups.isPrivilegeGroup(groupName);
});
if (!groupNames.length) {
return callback();
}
async.waterfall([
function (next) {
user.getUserData(uid, next);
},
function (userData, next) {
var newTitleArray = userData.groupTitleArray.filter(function (groupTitle) {
return !groupNames.includes(groupTitle);
});
if (newTitleArray.length) {
db.setObjectField('user:' + uid, 'groupTitle', JSON.stringify(newTitleArray), next);
} else {
db.deleteObjectField('user:' + uid, 'groupTitle', next);
}
},
], callback);
}
Groups.leaveAllGroups = function (uid, callback) {
async.waterfall([
function (next) {
db.getSortedSetRange('groups:createtime', 0, -1, next);
},
function (groups, next) {
async.parallel([
function (next) {
Groups.leave(groups, uid, next);
},
function (next) {
Groups.rejectMembership(groups, uid, next);
},
], next);
},
], callback);
};
};

@ -1,7 +1,6 @@
'use strict';
var async = require('async');
var winston = require('winston');
var _ = require('lodash');
var user = require('../user');
@ -10,105 +9,7 @@ var plugins = require('../plugins');
var notifications = require('../notifications');
var db = require('../database');
var pubsub = require('../pubsub');
var LRU = require('lru-cache');
var cache = LRU({
max: 40000,
maxAge: 0,
});
module.exports = function (Groups) {
Groups.cache = cache;
Groups.join = function (groupName, uid, callback) {
callback = callback || function () {};
if (!groupName) {
return callback(new Error('[[error:invalid-data]]'));
}
if (!uid) {
return callback(new Error('[[error:invalid-uid]]'));
}
async.waterfall([
function (next) {
Groups.isMember(uid, groupName, next);
},
function (isMember, next) {
if (isMember) {
return callback();
}
Groups.exists(groupName, next);
},
function (exists, next) {
if (exists) {
return next();
}
Groups.create({
name: groupName,
description: '',
hidden: 1,
}, function (err) {
if (err && err.message !== '[[error:group-already-exists]]') {
winston.error('[groups.join] Could not create new hidden group', err);
return callback(err);
}
next();
});
},
function (next) {
async.parallel({
isAdmin: function (next) {
user.isAdministrator(uid, next);
},
isHidden: function (next) {
Groups.isHidden(groupName, next);
},
}, next);
},
function (results, next) {
var tasks = [
async.apply(db.sortedSetAdd, 'group:' + groupName + ':members', Date.now(), uid),
async.apply(db.incrObjectField, 'group:' + groupName, 'memberCount'),
];
if (results.isAdmin) {
tasks.push(async.apply(db.setAdd, 'group:' + groupName + ':owners', uid));
}
if (!results.isHidden) {
tasks.push(async.apply(db.sortedSetIncrBy, 'groups:visible:memberCount', 1, groupName));
}
async.parallel(tasks, next);
},
function (results, next) {
clearCache(uid, groupName);
setGroupTitleIfNotSet(groupName, uid, next);
},
function (next) {
plugins.fireHook('action:group.join', {
groupName: groupName,
uid: uid,
});
next();
},
], callback);
};
function setGroupTitleIfNotSet(groupName, uid, callback) {
if (groupName === 'registered-users' || Groups.isPrivilegeGroup(groupName)) {
return callback();
}
db.getObjectField('user:' + uid, 'groupTitle', function (err, currentTitle) {
if (err || (currentTitle || currentTitle === '')) {
return callback(err);
}
user.setUserField(uid, 'groupTitle', JSON.stringify([groupName]), callback);
});
}
Groups.requestMembership = function (groupName, uid, callback) {
async.waterfall([
async.apply(inviteOrRequestMembership, groupName, uid, 'request'),
@ -214,120 +115,6 @@ module.exports = function (Groups) {
], callback);
}
Groups.leave = function (groupNames, uid, callback) {
callback = callback || function () {};
if (!Array.isArray(groupNames)) {
groupNames = [groupNames];
}
async.waterfall([
function (next) {
async.parallel({
isMembers: async.apply(Groups.isMemberOfGroups, uid, groupNames),
exists: async.apply(Groups.exists, groupNames),
}, next);
},
function (result, next) {
groupNames = groupNames.filter(function (groupName, index) {
return result.isMembers[index] && result.exists[index];
});
if (!groupNames.length) {
return callback();
}
async.parallel([
async.apply(db.sortedSetRemove, groupNames.map(groupName => 'group:' + groupName + ':members'), uid),
async.apply(db.setRemove, groupNames.map(groupName => 'group:' + groupName + ':owners'), uid),
async.apply(db.decrObjectField, groupNames.map(groupName => 'group:' + groupName), 'memberCount'),
], next);
},
function (results, next) {
clearCache(uid, groupNames);
Groups.getGroupsFields(groupNames, ['name', 'hidden', 'memberCount'], next);
},
function (groupData, next) {
if (!groupData) {
return callback();
}
var tasks = [];
var emptyPrivilegeGroups = groupData.filter(function (groupData) {
return groupData && Groups.isPrivilegeGroup(groupData.name) && parseInt(groupData.memberCount, 10) === 0;
});
if (emptyPrivilegeGroups.length) {
tasks.push(async.apply(Groups.destroy, emptyPrivilegeGroups));
}
var visibleGroups = groupData.filter(function (groupData) {
return groupData && parseInt(groupData.hidden, 10) !== 1;
});
if (visibleGroups.length) {
tasks.push(async.apply(db.sortedSetAdd, 'groups:visible:memberCount', visibleGroups.map(groupData => groupData.memberCount), visibleGroups.map(groupData => groupData.name)));
}
async.parallel(tasks, function (err) {
next(err);
});
},
function (next) {
clearGroupTitleIfSet(groupNames, uid, next);
},
function (next) {
plugins.fireHook('action:group.leave', {
groupName: groupNames[0],
groupNames: groupNames,
uid: uid,
});
next();
},
], callback);
};
function clearGroupTitleIfSet(groupNames, uid, callback) {
groupNames = groupNames.filter(function (groupName) {
return groupName !== 'registered-users' && !Groups.isPrivilegeGroup(groupName);
});
if (!groupNames.length) {
return callback();
}
async.waterfall([
function (next) {
user.getUserData(uid, next);
},
function (userData, next) {
var newTitleArray = userData.groupTitleArray.filter(function (groupTitle) {
return !groupNames.includes(groupTitle);
});
if (newTitleArray.length) {
db.setObjectField('user:' + uid, 'groupTitle', JSON.stringify(newTitleArray), next);
} else {
db.deleteObjectField('user:' + uid, 'groupTitle', next);
}
},
], callback);
}
Groups.leaveAllGroups = function (uid, callback) {
async.waterfall([
function (next) {
db.getSortedSetRange('groups:createtime', 0, -1, next);
},
function (groups, next) {
async.parallel([
function (next) {
Groups.leave(groups, uid, next);
},
function (next) {
Groups.rejectMembership(groups, uid, next);
},
], next);
},
], callback);
};
Groups.getMembers = function (groupName, start, stop, callback) {
db.getSortedSetRevRange('group:' + groupName + ':members', start, stop, callback);
};
@ -351,50 +138,24 @@ module.exports = function (Groups) {
}), callback);
};
Groups.resetCache = function () {
pubsub.publish('group:cache:reset');
cache.reset();
};
pubsub.on('group:cache:reset', function () {
cache.reset();
});
function clearCache(uid, groupNames) {
if (!Array.isArray(groupNames)) {
groupNames = [groupNames];
}
pubsub.publish('group:cache:del', { uid: uid, groupNames: groupNames });
groupNames.forEach(function (groupName) {
cache.del(uid + ':' + groupName);
});
}
pubsub.on('group:cache:del', function (data) {
if (data && data.groupNames) {
data.groupNames.forEach(function (groupName) {
cache.del(data.uid + ':' + groupName);
});
}
});
Groups.isMember = function (uid, groupName, callback) {
if (!uid || parseInt(uid, 10) <= 0 || !groupName) {
return setImmediate(callback, null, false);
}
var cacheKey = uid + ':' + groupName;
var isMember = cache.get(cacheKey);
var isMember = Groups.cache.get(cacheKey);
if (isMember !== undefined) {
Groups.cache.hits += 1;
return setImmediate(callback, null, isMember);
}
Groups.cache.misses += 1;
async.waterfall([
function (next) {
db.isSortedSetMember('group:' + groupName + ':members', uid, next);
},
function (isMember, next) {
cache.set(cacheKey, isMember);
Groups.cache.set(cacheKey, isMember);
next(null, isMember);
},
], callback);
@ -413,11 +174,7 @@ module.exports = function (Groups) {
}
var nonCachedUids = uids.filter(function (uid) {
var isMember = cache.get(uid + ':' + groupName);
if (isMember !== undefined) {
cachedData[uid + ':' + groupName] = isMember;
}
return isMember === undefined;
return filterNonCached(cachedData, uid, groupName);
});
if (!nonCachedUids.length) {
@ -431,7 +188,7 @@ module.exports = function (Groups) {
function (isMembers, next) {
nonCachedUids.forEach(function (uid, index) {
cachedData[uid + ':' + groupName] = isMembers[index];
cache.set(uid + ':' + groupName, isMembers[index]);
Groups.cache.set(uid + ':' + groupName, isMembers[index]);
});
getFromCache(next);
@ -439,6 +196,18 @@ module.exports = function (Groups) {
], callback);
};
function filterNonCached(cachedData, uid, groupName) {
var isMember = Groups.cache.get(uid + ':' + groupName);
var isInCache = isMember !== undefined;
if (isInCache) {
Groups.cache.hits += 1;
cachedData[uid + ':' + groupName] = isMember;
} else {
Groups.cache.misses += 1;
}
return !isInCache;
}
Groups.isMemberOfGroups = function (uid, groups, callback) {
var cachedData = {};
function getFromCache(next) {
@ -452,11 +221,7 @@ module.exports = function (Groups) {
}
var nonCachedGroups = groups.filter(function (groupName) {
var isMember = cache.get(uid + ':' + groupName);
if (isMember !== undefined) {
cachedData[uid + ':' + groupName] = isMember;
}
return isMember === undefined;
return filterNonCached(cachedData, uid, groupName);
});
if (!nonCachedGroups.length) {
@ -474,7 +239,7 @@ module.exports = function (Groups) {
function (isMembers, next) {
nonCachedGroups.forEach(function (groupName, index) {
cachedData[uid + ':' + groupName] = isMembers[index];
cache.set(uid + ':' + groupName, isMembers[index]);
Groups.cache.set(uid + ':' + groupName, isMembers[index]);
});
getFromCache(next);

@ -227,8 +227,16 @@ helpers.getGroupPrivileges = function (cid, hookName, groupPrivilegeList, callba
], callback);
};
helpers.giveOrRescind = function (method, privileges, cid, groupName, callback) {
async.eachSeries(privileges, function (privilege, next) {
method('cid:' + cid + ':privileges:groups:' + privilege, groupName, next);
helpers.giveOrRescind = function (method, privileges, cids, groupNames, callback) {
groupNames = Array.isArray(groupNames) ? groupNames : [groupNames];
cids = Array.isArray(cids) ? cids : [cids];
async.eachSeries(groupNames, function (groupName, next) {
var groupKeys = [];
cids.forEach((cid) => {
privileges.forEach((privilege) => {
groupKeys.push('cid:' + cid + ':privileges:groups:' + privilege);
});
});
method(groupKeys, groupName, next);
}, callback);
};

@ -29,9 +29,6 @@
<div class="panel panel-default">
<div class="panel-heading"><i class="fa fa-calendar-o"></i> Object Cache</div>
<div class="panel-body">
<label>[[admin/advanced/cache:length-to-max]]</label><br/>
<span>{objectCache.length} / {objectCache.max}</span><br/>
<div class="progress">
@ -63,6 +60,10 @@
</div>
</div>
<label>Hits:</label> <span>{groupCache.hits}</span><br/>
<label>Misses:</label> <span>{groupCache.misses}</span><br/>
<label>Hit Ratio:</label> <span>{groupCache.hitRatio}</span><br/>
<!-- IF groupCache.dump -->
<pre>{groupCache.dump}</pre>
<!-- ENDIF groupCache.dump -->

@ -486,6 +486,45 @@ describe('Groups', function () {
});
});
});
it('should add user to multiple groups', function (done) {
var groupNames = ['test-hidden1', 'Test', 'test-hidden2', 'empty group'];
Groups.create({ name: 'empty group' }, function (err) {
assert.ifError(err);
Groups.join(groupNames, testUid, function (err) {
assert.ifError(err);
Groups.isMemberOfGroups(testUid, groupNames, function (err, isMembers) {
assert.ifError(err);
assert(isMembers.every(Boolean));
db.sortedSetScores('groups:visible:memberCount', groupNames, function (err, memberCounts) {
assert.ifError(err);
// hidden groups are not in "groups:visible:memberCount" so they are null
assert.deepEqual(memberCounts, [null, 3, null, 1]);
done();
});
});
});
});
});
it('should set group title when user joins the group', function (done) {
var groupName = 'this will be title';
User.create({ username: 'needstitle' }, function (err, uid) {
assert.ifError(err);
Groups.create({ name: groupName }, function (err) {
assert.ifError(err);
Groups.join([groupName], uid, function (err) {
assert.ifError(err);
User.getUserData(uid, function (err, data) {
assert.ifError(err);
assert.equal(data.groupTitle, '["' + groupName + '"]');
assert.deepEqual(data.groupTitleArray, [groupName]);
done();
});
});
});
});
});
});
describe('.leave()', function () {

Loading…
Cancel
Save