diff --git a/public/language/en-GB/admin/advanced/cache.json b/public/language/en-GB/admin/advanced/cache.json index 5a954f1232..cbbae7ad19 100644 --- a/public/language/en-GB/admin/advanced/cache.json +++ b/public/language/en-GB/admin/advanced/cache.json @@ -1,11 +1,11 @@ { "post-cache": "Post Cache", - "posts-in-cache": "Posts in Cache", - "average-post-size": "Average Post Size", - "length-to-max": "Length / Max", "percent-full": "%1% Full", "post-cache-size": "Post Cache Size", "items-in-cache": "Items in Cache", "control-panel": "Control Panel", - "update-settings": "Update Cache Settings" + "update-settings": "Update Cache Settings", + "clear": "Clear", + "download": "Download", + "enabled": "Enabled" } \ No newline at end of file diff --git a/public/openapi/read/admin/advanced/cache.yaml b/public/openapi/read/admin/advanced/cache.yaml index 4d54740eb9..06d7890971 100644 --- a/public/openapi/read/admin/advanced/cache.yaml +++ b/public/openapi/read/admin/advanced/cache.yaml @@ -23,14 +23,14 @@ get: type: number percentFull: type: number - avgPostSize: - type: number hits: type: string misses: type: string hitRatio: type: string + enabled: + type: boolean groupCache: type: object properties: @@ -48,6 +48,8 @@ get: type: string hitRatio: type: string + enabled: + type: boolean localCache: type: object properties: @@ -59,14 +61,14 @@ get: type: number percentFull: type: number - dump: - type: boolean hits: type: string misses: type: string hitRatio: type: string + enabled: + type: boolean objectCache: type: object properties: @@ -84,6 +86,8 @@ get: type: string hitRatio: type: string + enabled: + type: boolean required: - postCache - groupCache diff --git a/public/src/admin/advanced/cache.js b/public/src/admin/advanced/cache.js index 55eaf13d7f..f1cab9af09 100644 --- a/public/src/admin/advanced/cache.js +++ b/public/src/admin/advanced/cache.js @@ -15,6 +15,16 @@ define('admin/advanced/cache', function () { ajaxify.refresh(); }); }); + $('.checkbox').on('change', function () { + var input = $(this).find('input'); + var flag = input.is(':checked'); + var name = $(this).attr('data-name'); + socket.emit('admin.cache.toggle', { name: name, enabled: flag }, function (err) { + if (err) { + return app.alertError(err.message); + } + }); + }); }; return Cache; }); diff --git a/src/cache.js b/src/cache.js index 7fd9c6492c..a77cc6e637 100644 --- a/src/cache.js +++ b/src/cache.js @@ -1,56 +1,9 @@ 'use strict'; -const LRU = require('lru-cache'); -const pubsub = require('./pubsub'); +const cacheCreate = require('./cacheCreate'); -const cache = new LRU({ +module.exports = cacheCreate({ + name: 'local', max: 4000, maxAge: 0, }); -cache.hits = 0; -cache.misses = 0; - -const cacheGet = cache.get; -const cacheDel = cache.del; -const cacheReset = cache.reset; - -cache.get = function (key) { - const data = cacheGet.apply(cache, [key]); - if (data === undefined) { - cache.misses += 1; - } else { - cache.hits += 1; - } - return data; -}; - -cache.del = function (key) { - if (!Array.isArray(key)) { - key = [key]; - } - pubsub.publish('local:cache:del', key); - key.forEach(key => cacheDel.apply(cache, [key])); -}; - -cache.reset = function () { - pubsub.publish('local:cache:reset'); - localReset(); -}; - -function localReset() { - cacheReset.apply(cache); - cache.hits = 0; - cache.misses = 0; -} - -pubsub.on('local:cache:reset', function () { - localReset(); -}); - -pubsub.on('local:cache:del', function (keys) { - if (Array.isArray(keys)) { - keys.forEach(key => cacheDel.apply(cache, [key])); - } -}); - -module.exports = cache; diff --git a/src/cacheCreate.js b/src/cacheCreate.js new file mode 100644 index 0000000000..0543b691f5 --- /dev/null +++ b/src/cacheCreate.js @@ -0,0 +1,91 @@ +'use strict'; + +module.exports = function (opts) { + const LRU = require('lru-cache'); + const pubsub = require('./pubsub'); + + const cache = new LRU(opts); + + cache.name = opts.name; + cache.hits = 0; + cache.misses = 0; + cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true; + + const cacheSet = cache.set; + const cacheGet = cache.get; + const cacheDel = cache.del; + const cacheReset = cache.reset; + + cache.set = function (key, value) { + if (!cache.enabled) { + return; + } + cacheSet.apply(cache, [key, value]); + }; + + cache.get = function (key) { + if (!cache.enabled) { + return undefined; + } + const data = cacheGet.apply(cache, [key]); + if (data === undefined) { + cache.misses += 1; + } else { + cache.hits += 1; + } + return data; + }; + + cache.del = function (keys) { + if (!Array.isArray(keys)) { + keys = [keys]; + } + pubsub.publish(cache.name + ':cache:del', keys); + keys.forEach(key => cacheDel.apply(cache, [key])); + }; + + cache.reset = function () { + pubsub.publish(cache.name + ':cache:reset'); + localReset(); + }; + + function localReset() { + cacheReset.apply(cache); + cache.hits = 0; + cache.misses = 0; + } + + pubsub.on(cache.name + ':cache:reset', function () { + localReset(); + }); + + pubsub.on(cache.name + ':cache:del', function (keys) { + if (Array.isArray(keys)) { + keys.forEach(key => cacheDel.apply(cache, [key])); + } + }); + + cache.getUnCachedKeys = function (keys, cachedData) { + if (!cache.enabled) { + return keys; + } + let data; + let isCached; + const unCachedKeys = keys.filter(function (key) { + data = cache.get(key); + isCached = data !== undefined; + if (isCached) { + cachedData[key] = data; + } + return !isCached; + }); + + var hits = keys.length - unCachedKeys.length; + var misses = keys.length - hits; + cache.hits += hits; + cache.misses += misses; + return unCachedKeys; + }; + + return cache; +}; diff --git a/src/controllers/admin/cache.js b/src/controllers/admin/cache.js index 165e4537b4..e96dfffec3 100644 --- a/src/controllers/admin/cache.js +++ b/src/controllers/admin/cache.js @@ -10,10 +10,8 @@ cacheController.get = function (req, res) { const objectCache = require('../../database').objectCache; const localCache = require('../../cache'); - let avgPostSize = 0; let percentFull = 0; if (postCache.itemCount > 0) { - avgPostSize = parseInt((postCache.length / postCache.itemCount), 10); percentFull = ((postCache.length / postCache.max) * 100).toFixed(2); } @@ -23,10 +21,10 @@ cacheController.get = function (req, res) { max: postCache.max, itemCount: postCache.itemCount, percentFull: percentFull, - avgPostSize: avgPostSize, hits: utils.addCommas(String(postCache.hits)), misses: utils.addCommas(String(postCache.misses)), hitRatio: ((postCache.hits / (postCache.hits + postCache.misses) || 0)).toFixed(4), + enabled: postCache.enabled, }, groupCache: { length: groupCache.length, @@ -36,16 +34,17 @@ cacheController.get = function (req, res) { hits: utils.addCommas(String(groupCache.hits)), misses: utils.addCommas(String(groupCache.misses)), hitRatio: (groupCache.hits / (groupCache.hits + groupCache.misses)).toFixed(4), + enabled: groupCache.enabled, }, localCache: { length: localCache.length, max: localCache.max, itemCount: localCache.itemCount, percentFull: ((localCache.length / localCache.max) * 100).toFixed(2), - dump: req.query.debug ? JSON.stringify(localCache.dump(), null, 4) : false, hits: utils.addCommas(String(localCache.hits)), misses: utils.addCommas(String(localCache.misses)), hitRatio: ((localCache.hits / (localCache.hits + localCache.misses) || 0)).toFixed(4), + enabled: localCache.enabled, }, }; @@ -58,8 +57,31 @@ cacheController.get = function (req, res) { hits: utils.addCommas(String(objectCache.hits)), misses: utils.addCommas(String(objectCache.misses)), hitRatio: (objectCache.hits / (objectCache.hits + objectCache.misses)).toFixed(4), + enabled: objectCache.enabled, }; } res.render('admin/advanced/cache', data); }; + +cacheController.dump = function (req, res, next) { + const caches = { + post: require('../../posts/cache'), + object: require('../../database').objectCache, + group: require('../../groups').cache, + local: require('../../cache'), + }; + if (!caches[req.query.name]) { + return next(); + } + + const data = JSON.stringify(caches[req.query.name].dump(), null, 4); + res.setHeader('Content-disposition', 'attachment; filename= ' + req.query.name + '-cache.json'); + res.setHeader('Content-type', 'application/json'); + res.write(data, function (err) { + if (err) { + return next(err); + } + res.end(); + }); +}; diff --git a/src/database/cache.js b/src/database/cache.js index c013443f6a..1e657d6b62 100644 --- a/src/database/cache.js +++ b/src/database/cache.js @@ -1,56 +1,11 @@ 'use strict'; module.exports.create = function (name) { - var LRU = require('lru-cache'); - var pubsub = require('../pubsub'); - - var cache = new LRU({ + const cacheCreate = require('../cacheCreate'); + return cacheCreate({ + name: name + '-object', max: 40000, length: function () { return 1; }, maxAge: 0, }); - - cache.misses = 0; - cache.hits = 0; - - pubsub.on(name + ':hash:cache:del', function (keys) { - keys.forEach(key => cache.del(key)); - }); - - pubsub.on(name + ':hash:cache:reset', function () { - cache.reset(); - }); - - cache.delObjectCache = function (keys) { - if (!Array.isArray(keys)) { - keys = [keys]; - } - pubsub.publish(name + ':hash:cache:del', keys); - keys.forEach(key => cache.del(key)); - }; - - cache.resetObjectCache = function () { - pubsub.publish(name + ':hash:cache:reset'); - cache.reset(); - }; - - cache.getUnCachedKeys = function (keys, cachedData) { - let data; - let isCached; - const unCachedKeys = keys.filter(function (key) { - data = cache.get(key); - isCached = data !== undefined; - if (isCached) { - cachedData[key] = data; - } - return !isCached; - }); - - var hits = keys.length - unCachedKeys.length; - var misses = keys.length - hits; - cache.hits += hits; - cache.misses += misses; - return unCachedKeys; - }; - return cache; }; diff --git a/src/database/mongo/hash.js b/src/database/mongo/hash.js index 59d6ee2585..58db98c537 100644 --- a/src/database/mongo/hash.js +++ b/src/database/mongo/hash.js @@ -29,7 +29,7 @@ module.exports = function (module) { throw err; } - cache.delObjectCache(key); + cache.del(key); }; module.setObjectField = async function (key, field, value) { @@ -164,7 +164,7 @@ module.exports = function (module) { }); await module.client.collection('objects').updateOne({ _key: key }, { $unset: data }); - cache.delObjectCache(key); + cache.del(key); }; module.incrObjectField = async function (key, field) { @@ -191,13 +191,13 @@ module.exports = function (module) { bulk.find({ _key: key }).upsert().update({ $inc: increment }); }); await bulk.execute(); - cache.delObjectCache(key); + cache.del(key); const result = await module.getObjectsFields(key, [field]); return result.map(data => data && data[field]); } const result = await module.client.collection('objects').findOneAndUpdate({ _key: key }, { $inc: increment }, { returnOriginal: false, upsert: true }); - cache.delObjectCache(key); + cache.del(key); return result && result.value ? result.value[field] : null; }; }; diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js index 67a209dfdb..a1cba7b1e7 100644 --- a/src/database/mongo/main.js +++ b/src/database/mongo/main.js @@ -8,7 +8,7 @@ module.exports = function (module) { module.emptydb = async function () { await module.client.collection('objects').deleteMany({}); - module.objectCache.resetObjectCache(); + module.objectCache.reset(); }; module.exists = async function (key) { @@ -47,7 +47,7 @@ module.exports = function (module) { return; } await module.client.collection('objects').deleteMany({ _key: key }); - module.objectCache.delObjectCache(key); + module.objectCache.del(key); }; module.deleteAll = async function (keys) { @@ -55,7 +55,7 @@ module.exports = function (module) { return; } await module.client.collection('objects').deleteMany({ _key: { $in: keys } }); - module.objectCache.delObjectCache(keys); + module.objectCache.del(keys); }; module.get = async function (key) { @@ -96,7 +96,7 @@ module.exports = function (module) { module.rename = async function (oldKey, newKey) { await module.client.collection('objects').updateMany({ _key: oldKey }, { $set: { _key: newKey } }); - module.objectCache.delObjectCache([oldKey, newKey]); + module.objectCache.del([oldKey, newKey]); }; module.type = async function (key) { diff --git a/src/database/redis/hash.js b/src/database/redis/hash.js index 0426485892..f32f7be0e5 100644 --- a/src/database/redis/hash.js +++ b/src/database/redis/hash.js @@ -33,7 +33,7 @@ module.exports = function (module) { await module.client.async.hmset(key, data); } - cache.delObjectCache(key); + cache.del(key); }; module.setObjectField = async function (key, field, value) { @@ -48,7 +48,7 @@ module.exports = function (module) { await module.client.async.hset(key, field, value); } - cache.delObjectCache(key); + cache.del(key); }; module.getObject = async function (key) { @@ -146,7 +146,7 @@ module.exports = function (module) { return; } await module.client.async.hdel(key, field); - cache.delObjectCache(key); + cache.del(key); }; module.deleteObjectFields = async function (key, fields) { @@ -158,7 +158,7 @@ module.exports = function (module) { return; } await module.client.async.hdel(key, fields); - cache.delObjectCache(key); + cache.del(key); }; module.incrObjectField = async function (key, field) { @@ -182,7 +182,7 @@ module.exports = function (module) { } else { result = await module.client.async.hincrby(key, field, value); } - cache.delObjectCache(key); + cache.del(key); return Array.isArray(result) ? result.map(value => parseInt(value, 10)) : parseInt(result, 10); }; }; diff --git a/src/database/redis/main.js b/src/database/redis/main.js index 5f31ee2da2..0d98058d36 100644 --- a/src/database/redis/main.js +++ b/src/database/redis/main.js @@ -9,7 +9,7 @@ module.exports = function (module) { module.emptydb = async function () { await module.flushdb(); - module.objectCache.resetObjectCache(); + module.objectCache.reset(); }; module.exists = async function (key) { @@ -45,7 +45,7 @@ module.exports = function (module) { module.delete = async function (key) { await module.client.async.del(key); - module.objectCache.delObjectCache(key); + module.objectCache.del(key); }; module.deleteAll = async function (keys) { @@ -53,7 +53,7 @@ module.exports = function (module) { return; } await module.client.async.del(keys); - module.objectCache.delObjectCache(keys); + module.objectCache.del(keys); }; module.get = async function (key) { @@ -77,7 +77,7 @@ module.exports = function (module) { } } - module.objectCache.delObjectCache([oldKey, newKey]); + module.objectCache.del([oldKey, newKey]); }; module.type = async function (key) { diff --git a/src/groups/cache.js b/src/groups/cache.js index 70e4f85472..642b9466f6 100644 --- a/src/groups/cache.js +++ b/src/groups/cache.js @@ -1,48 +1,19 @@ 'use strict'; -var LRU = require('lru-cache'); -var pubsub = require('../pubsub'); - -var cache = new LRU({ - max: 40000, - maxAge: 0, -}); -cache.hits = 0; -cache.misses = 0; +const cacheCreate = require('../cacheCreate'); module.exports = function (Groups) { - Groups.cache = cache; - - pubsub.on('group:cache:reset', function () { - localReset(); - }); - - pubsub.on('group:cache:del', function (data) { - if (data && data.groupNames) { - data.groupNames.forEach(function (groupName) { - cache.del(data.uid + ':' + groupName); - }); - } + Groups.cache = cacheCreate({ + name: 'group', + max: 40000, + maxAge: 0, }); - Groups.resetCache = function () { - pubsub.publish('group:cache:reset'); - localReset(); - }; - - function localReset() { - cache.reset(); - cache.hits = 0; - cache.misses = 0; - } - Groups.clearCache = function (uid, groupNames) { if (!Array.isArray(groupNames)) { groupNames = [groupNames]; } - pubsub.publish('group:cache:del', { uid: uid, groupNames: groupNames }); - groupNames.forEach(function (groupName) { - cache.del(uid + ':' + groupName); - }); + const keys = groupNames.map(name => uid + ':' + name); + Groups.cache.del(keys); }; }; diff --git a/src/groups/delete.js b/src/groups/delete.js index 1bf3068f5a..eb6ef3b64e 100644 --- a/src/groups/delete.js +++ b/src/groups/delete.js @@ -40,7 +40,7 @@ module.exports = function (Groups) { db.deleteObjectFields('groupslug:groupname', fields), removeGroupsFromPrivilegeGroups(groupNames), ]); - Groups.resetCache(); + Groups.cache.reset(); plugins.fireHook('action:groups.destroy', { groups: groupsData }); }; diff --git a/src/groups/membership.js b/src/groups/membership.js index 7739445398..8d62e6a117 100644 --- a/src/groups/membership.js +++ b/src/groups/membership.js @@ -31,10 +31,8 @@ module.exports = function (Groups) { const cacheKey = uid + ':' + groupName; let isMember = Groups.cache.get(cacheKey); if (isMember !== undefined) { - Groups.cache.hits += 1; return isMember; } - Groups.cache.misses += 1; isMember = await db.isSortedSetMember('group:' + groupName + ':members', uid); Groups.cache.set(cacheKey, isMember); return isMember; @@ -88,10 +86,7 @@ module.exports = function (Groups) { const isMember = Groups.cache.get(uid + ':' + groupName); const isInCache = isMember !== undefined; if (isInCache) { - Groups.cache.hits += 1; cachedData[uid + ':' + groupName] = isMember; - } else { - Groups.cache.misses += 1; } return !isInCache; } diff --git a/src/groups/update.js b/src/groups/update.js index e83c016b82..eb477944c5 100644 --- a/src/groups/update.js +++ b/src/groups/update.js @@ -197,7 +197,7 @@ module.exports = function (Groups) { old: oldName, new: newName, }); - Groups.resetCache(); + Groups.cache.reset(); }; async function updateMemberGroupTitles(oldName, newName) { diff --git a/src/posts/cache.js b/src/posts/cache.js index 4d03cd3db1..061c46d88d 100644 --- a/src/posts/cache.js +++ b/src/posts/cache.js @@ -1,14 +1,12 @@ 'use strict'; -var LRU = require('lru-cache'); -var meta = require('../meta'); +const cacheCreate = require('../cacheCreate'); +const meta = require('../meta'); -var cache = new LRU({ +module.exports = cacheCreate({ + name: 'post', max: meta.config.postCacheSize, length: function (n) { return n.length; }, maxAge: 0, + enabled: global.env === 'production', }); -cache.hits = 0; -cache.misses = 0; - -module.exports = cache; diff --git a/src/posts/parse.js b/src/posts/parse.js index b07a39a64f..6b2d59600e 100644 --- a/src/posts/parse.js +++ b/src/posts/parse.js @@ -57,13 +57,12 @@ module.exports = function (Posts) { const cachedContent = cache.get(pid); if (postData.pid && cachedContent !== undefined) { postData.content = cachedContent; - cache.hits += 1; return postData; } - cache.misses += 1; + const data = await plugins.fireHook('filter:parse.post', { postData: postData }); data.postData.content = translator.escape(data.postData.content); - if (global.env === 'production' && data.postData.pid) { + if (data.postData.pid) { cache.set(pid, data.postData.content); } return data.postData; diff --git a/src/privileges/helpers.js b/src/privileges/helpers.js index ab596bf395..9d1fb78e61 100644 --- a/src/privileges/helpers.js +++ b/src/privileges/helpers.js @@ -41,6 +41,9 @@ helpers.isUserAllowedTo = async function (privilege, uid, cid) { }; async function isUserAllowedToCids(privilege, uid, cids) { + if (!privilege) { + return cids.map(() => false); + } if (parseInt(uid, 10) <= 0) { return await isSystemGroupAllowedToCids(privilege, uid, cids); } diff --git a/src/routes/admin.js b/src/routes/admin.js index e528a15068..bd017c4124 100644 --- a/src/routes/admin.js +++ b/src/routes/admin.js @@ -61,6 +61,7 @@ function apiRoutes(router, middleware, controllers) { router.get('/api/admin/users/csv', middleware.authenticate, helpers.tryRoute(controllers.admin.users.getCSV)); router.get('/api/admin/groups/:groupname/csv', middleware.authenticate, helpers.tryRoute(controllers.admin.groups.getCSV)); router.get('/api/admin/analytics', middleware.authenticate, helpers.tryRoute(controllers.admin.dashboard.getAnalytics)); + router.get('/api/admin/advanced/cache/dump', middleware.authenticate, helpers.tryRoute(controllers.admin.cache.dump)); const multipart = require('connect-multiparty'); const multipartMiddleware = multipart(); diff --git a/src/socket.io/admin/cache.js b/src/socket.io/admin/cache.js index 2852e1dd3c..ad057534cb 100644 --- a/src/socket.io/admin/cache.js +++ b/src/socket.io/admin/cache.js @@ -8,3 +8,16 @@ SocketCache.clear = async function () { require('../../groups').cache.reset(); require('../../cache').reset(); }; + +SocketCache.toggle = async function (socket, data) { + const caches = { + post: require('../../posts/cache'), + object: require('../../database').objectCache, + group: require('../../groups').cache, + local: require('../../cache'), + }; + if (!caches[data.name]) { + return; + } + caches[data.name].enabled = data.enabled; +}; diff --git a/src/views/admin/advanced/cache.tpl b/src/views/admin/advanced/cache.tpl index dfc7caa9b0..972491291e 100644 --- a/src/views/admin/advanced/cache.tpl +++ b/src/views/admin/advanced/cache.tpl @@ -6,7 +6,13 @@
[[admin/advanced/cache:post-cache]]
-
+
+ +
+ {postCache.length} / {postCache.max}
@@ -21,16 +27,11 @@
-
- {postCache.itemCount}
- -
- {postCache.avgPostSize}
-
+ [[admin/advanced/cache:download]]
@@ -40,7 +41,12 @@
Object Cache
-
+
+ +
{objectCache.length} / {objectCache.max}
@@ -51,6 +57,7 @@ {objectCache.hits}
{objectCache.misses}
{objectCache.hitRatio}
+ [[admin/advanced/cache:download]]
@@ -59,8 +66,12 @@
Group Cache
- -
+
+ +
{groupCache.length} / {groupCache.max}
@@ -72,6 +83,7 @@ {groupCache.hits}
{groupCache.misses}
{groupCache.hitRatio}
+ [[admin/advanced/cache:download]]
@@ -79,8 +91,12 @@
Local Cache
- -
+
+ +
{localCache.length} / {localCache.max}
@@ -92,11 +108,7 @@ {localCache.hits}
{localCache.misses}
{localCache.hitRatio}
- - - 
{localCache.dump}
- - + [[admin/advanced/cache:download]]
diff --git a/test/meta.js b/test/meta.js index e244100e28..cb0c79a72e 100644 --- a/test/meta.js +++ b/test/meta.js @@ -16,7 +16,7 @@ describe('meta', function () { var herpUid; before(function (done) { - Groups.resetCache(); + Groups.cache.reset(); // Create 3 users: 1 admin, 2 regular async.series([ async.apply(User.create, { username: 'foo', password: 'barbar' }), // admin diff --git a/test/mocks/databasemock.js b/test/mocks/databasemock.js index e87feaf649..6ac086aff4 100644 --- a/test/mocks/databasemock.js +++ b/test/mocks/databasemock.js @@ -168,7 +168,7 @@ async function setupMockDefaults() { const meta = require('../../src/meta'); await db.emptydb(); - require('../../src/groups').resetCache(); + require('../../src/groups').cache.reset(); require('../../src/posts/cache').reset(); require('../../src/cache').reset(); winston.info('test_database flushed');