From c9754b09e67325ea44156b3675cc45c6fe890cda Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 10 Aug 2022 13:22:18 -0400 Subject: [PATCH 01/11] Update to lru-cache@^7 (#10813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): bump lru-cache from 6.0.0 to 7.13.1 in /install Bumps [lru-cache](https://github.com/isaacs/node-lru-cache) from 6.0.0 to 7.13.1. - [Release notes](https://github.com/isaacs/node-lru-cache/releases) - [Changelog](https://github.com/isaacs/node-lru-cache/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-lru-cache/compare/v6.0.0...v7.13.1) --- updated-dependencies: - dependency-name: lru-cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * fix(lru-cache): remove unneeded `length` params for cache creation, as `maxSize` was not used in those init calls, also renamed some methods to match new method names in lru-cache [breaking] Added deprecation notices for old params * fix: replace three direct calls to lru-cache with call to cacheCreate, moved cache creation call in uploads to run on first init as config is not populated at lib init * test: move configs init above cache reset calls in databasemock * move some more code above cache clear * refactor: remove unused * test: lru * test: more debug * test: on more test * use await helpers.uploadFile * fix: tests remove logs * fix: acp cache page * fix: add in one more guard again cache instantiation with `length` prop but no `maxSize` prop * fix(deps): bump markdown Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Barış Soner Uşaklı --- install/package.json | 6 +-- src/analytics.js | 7 ++- src/cache.js | 2 +- src/cacheCreate.js | 74 +++++++++++++++++++++++++----- src/controllers/admin/cache.js | 5 +- src/database/cache.js | 3 +- src/groups/cache.js | 2 +- src/middleware/index.js | 6 +-- src/middleware/uploads.js | 9 ++-- src/posts/cache.js | 6 +-- src/user/blocks.js | 3 +- src/views/admin/advanced/cache.tpl | 8 ++-- test/middleware.js | 7 ++- test/mocks/databasemock.js | 13 +++--- test/uploads.js | 5 +- 15 files changed, 106 insertions(+), 50 deletions(-) diff --git a/install/package.json b/install/package.json index 0a62a66e88..d27a13a5ee 100644 --- a/install/package.json +++ b/install/package.json @@ -77,7 +77,7 @@ "less": "4.1.3", "lodash": "4.17.21", "logrotate-stream": "0.2.8", - "lru-cache": "6.0.0", + "lru-cache": "7.13.1", "material-design-lite": "1.3.0", "mime": "3.0.0", "mkdirp": "1.0.4", @@ -92,7 +92,7 @@ "nodebb-plugin-dbsearch": "5.1.5", "nodebb-plugin-emoji": "4.0.4", "nodebb-plugin-emoji-android": "3.0.0", - "nodebb-plugin-markdown": "10.0.0", + "nodebb-plugin-markdown": "10.1.0", "nodebb-plugin-mentions": "3.0.11", "nodebb-plugin-spam-be-gone": "1.0.0", "nodebb-rewards-essentials": "0.2.1", @@ -186,4 +186,4 @@ "url": "https://github.com/barisusakli" } ] -} \ No newline at end of file +} diff --git a/src/analytics.js b/src/analytics.js index 5d71d161a8..f9e14b4caf 100644 --- a/src/analytics.js +++ b/src/analytics.js @@ -4,7 +4,6 @@ const cronJob = require('cron').CronJob; const winston = require('winston'); const nconf = require('nconf'); const crypto = require('crypto'); -const LRU = require('lru-cache'); const util = require('util'); const _ = require('lodash'); @@ -15,6 +14,7 @@ const utils = require('./utils'); const plugins = require('./plugins'); const meta = require('./meta'); const pubsub = require('./pubsub'); +const cacheCreate = require('./cacheCreate'); const Analytics = module.exports; @@ -37,10 +37,9 @@ let ipCache; const runJobs = nconf.get('runJobs'); Analytics.init = async function () { - ipCache = new LRU({ + ipCache = cacheCreate({ max: parseInt(meta.config['analytics:maxCache'], 10) || 500, - length: function () { return 1; }, - maxAge: 0, + ttl: 0, }); new cronJob('*/10 * * * * *', (async () => { diff --git a/src/cache.js b/src/cache.js index 7bb27961ce..6dcc440536 100644 --- a/src/cache.js +++ b/src/cache.js @@ -5,5 +5,5 @@ const cacheCreate = require('./cacheCreate'); module.exports = cacheCreate({ name: 'local', max: 40000, - maxAge: 0, + ttl: 0, }); diff --git a/src/cacheCreate.js b/src/cacheCreate.js index e3d7342165..deaef9bba4 100644 --- a/src/cacheCreate.js +++ b/src/cacheCreate.js @@ -4,30 +4,72 @@ module.exports = function (opts) { const LRU = require('lru-cache'); const pubsub = require('./pubsub'); - const cache = new LRU(opts); + // lru-cache@7 deprecations + const winston = require('winston'); + const chalk = require('chalk'); + // sometimes we kept passing in `length` with no corresponding `maxSize`. + // This is now enforced in v7; drop superfluous property + if (opts.hasOwnProperty('length') && !opts.hasOwnProperty('maxSize')) { + winston.warn(`[cache/init(${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} ${chalk.yellow('length')} was passed in without a corresponding ${chalk.yellow('maxSize')}. Both are now required as of lru-cache@7.0.0.`); + delete opts.length; + } + + const deprecations = new Map([ + ['stale', 'allowStale'], + ['maxAge', 'ttl'], + ['length', 'sizeCalculation'], + ]); + deprecations.forEach((newProp, oldProp) => { + if (opts.hasOwnProperty(oldProp) && !opts.hasOwnProperty(newProp)) { + winston.warn(`[cache/init (${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} The option ${chalk.yellow(oldProp)} has been deprecated as of lru-cache@7.0.0. Please change this to ${chalk.yellow(newProp)} instead.`); + opts[newProp] = opts[oldProp]; + delete opts[oldProp]; + } + }); + + const lruCache = new LRU(opts); + + const cache = {}; cache.name = opts.name; cache.hits = 0; cache.misses = 0; cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true; + const cacheSet = lruCache.set; - const cacheSet = cache.set; - const cacheGet = cache.get; - const cacheDel = cache.del; - const cacheReset = cache.reset; + // backwards compatibility + const propertyMap = new Map([ + ['length', 'calculatedSize'], + ['max', 'max'], + ['maxSize', 'maxSize'], + ['itemCount', 'size'], + ]); + propertyMap.forEach((lruProp, cacheProp) => { + Object.defineProperty(cache, cacheProp, { + get: function () { + return lruCache[lruProp]; + }, + configurable: true, + enumerable: true, + }); + }); - cache.set = function (key, value, maxAge) { + cache.set = function (key, value, ttl) { if (!cache.enabled) { return; } - cacheSet.apply(cache, [key, value, maxAge]); + const opts = {}; + if (ttl) { + opts.ttl = ttl; + } + cacheSet.apply(lruCache, [key, value, opts]); }; cache.get = function (key) { if (!cache.enabled) { return undefined; } - const data = cacheGet.apply(cache, [key]); + const data = lruCache.get(key); if (data === undefined) { cache.misses += 1; } else { @@ -41,16 +83,18 @@ module.exports = function (opts) { keys = [keys]; } pubsub.publish(`${cache.name}:cache:del`, keys); - keys.forEach(key => cacheDel.apply(cache, [key])); + keys.forEach(key => lruCache.delete(key)); }; + cache.delete = cache.del; cache.reset = function () { pubsub.publish(`${cache.name}:cache:reset`); localReset(); }; + cache.clear = cache.reset; function localReset() { - cacheReset.apply(cache); + lruCache.clear(); cache.hits = 0; cache.misses = 0; } @@ -61,7 +105,7 @@ module.exports = function (opts) { pubsub.on(`${cache.name}:cache:del`, (keys) => { if (Array.isArray(keys)) { - keys.forEach(key => cacheDel.apply(cache, [key])); + keys.forEach(key => lruCache.delete(key)); } }); @@ -87,5 +131,13 @@ module.exports = function (opts) { return unCachedKeys; }; + cache.dump = function () { + return lruCache.dump(); + }; + + cache.peek = function (key) { + return lruCache.peek(key); + }; + return cache; }; diff --git a/src/controllers/admin/cache.js b/src/controllers/admin/cache.js index 430cc28b8a..67c9864ae5 100644 --- a/src/controllers/admin/cache.js +++ b/src/controllers/admin/cache.js @@ -14,8 +14,11 @@ cacheController.get = function (req, res) { return { length: cache.length, max: cache.max, + maxSize: cache.maxSize, itemCount: cache.itemCount, - percentFull: ((cache.length / cache.max) * 100).toFixed(2), + percentFull: cache.name === 'post' ? + ((cache.length / cache.maxSize) * 100).toFixed(2) : + ((cache.itemCount / cache.max) * 100).toFixed(2), hits: utils.addCommas(String(cache.hits)), misses: utils.addCommas(String(cache.misses)), hitRatio: ((cache.hits / (cache.hits + cache.misses) || 0)).toFixed(4), diff --git a/src/database/cache.js b/src/database/cache.js index 069b181a80..999c860363 100644 --- a/src/database/cache.js +++ b/src/database/cache.js @@ -5,7 +5,6 @@ module.exports.create = function (name) { return cacheCreate({ name: `${name}-object`, max: 40000, - length: function () { return 1; }, - maxAge: 0, + ttl: 0, }); }; diff --git a/src/groups/cache.js b/src/groups/cache.js index 0acea646d6..b83e3e5928 100644 --- a/src/groups/cache.js +++ b/src/groups/cache.js @@ -6,7 +6,7 @@ module.exports = function (Groups) { Groups.cache = cacheCreate({ name: 'group', max: 40000, - maxAge: 0, + ttl: 0, }); Groups.clearCache = function (uid, groupNames) { diff --git a/src/middleware/index.js b/src/middleware/index.js index a31cc4430d..6930021aee 100644 --- a/src/middleware/index.js +++ b/src/middleware/index.js @@ -6,7 +6,6 @@ const csrf = require('csurf'); const validator = require('validator'); const nconf = require('nconf'); const toobusy = require('toobusy-js'); -const LRU = require('lru-cache'); const util = require('util'); const plugins = require('../plugins'); @@ -15,6 +14,7 @@ const user = require('../user'); const groups = require('../groups'); const analytics = require('../analytics'); const privileges = require('../privileges'); +const cacheCreate = require('../cacheCreate'); const helpers = require('./helpers'); const controllers = { @@ -22,8 +22,8 @@ const controllers = { helpers: require('../controllers/helpers'), }; -const delayCache = new LRU({ - maxAge: 1000 * 60, +const delayCache = cacheCreate({ + ttl: 1000 * 60, }); const middleware = module.exports; diff --git a/src/middleware/uploads.js b/src/middleware/uploads.js index 941b9c27bc..beb36c03e0 100644 --- a/src/middleware/uploads.js +++ b/src/middleware/uploads.js @@ -1,16 +1,16 @@ 'use strict'; -const LRU = require('lru-cache'); +const cacheCreate = require('../cacheCreate'); const meta = require('../meta'); const helpers = require('./helpers'); const user = require('../user'); -const cache = new LRU({ - maxAge: meta.config.uploadRateLimitCooldown * 1000, +const cache = cacheCreate({ + ttl: meta.config.uploadRateLimitCooldown * 1000, }); exports.clearCache = function () { - cache.reset(); + cache.clear(); }; exports.ratelimit = helpers.try(async (req, res, next) => { @@ -23,7 +23,6 @@ exports.ratelimit = helpers.try(async (req, res, next) => { if (count > meta.config.uploadRateLimitThreshold) { return next(new Error(['[[error:upload-ratelimit-reached]]'])); } - cache.set(`${req.ip}:uploaded_file_count`, count); next(); }); diff --git a/src/posts/cache.js b/src/posts/cache.js index 061c46d88d..246074697c 100644 --- a/src/posts/cache.js +++ b/src/posts/cache.js @@ -5,8 +5,8 @@ const meta = require('../meta'); module.exports = cacheCreate({ name: 'post', - max: meta.config.postCacheSize, - length: function (n) { return n.length; }, - maxAge: 0, + maxSize: meta.config.postCacheSize, + sizeCalculation: function (n) { return n.length; }, + ttl: 0, enabled: global.env === 'production', }); diff --git a/src/user/blocks.js b/src/user/blocks.js index 6655965f2a..f71589911c 100644 --- a/src/user/blocks.js +++ b/src/user/blocks.js @@ -9,8 +9,7 @@ module.exports = function (User) { _cache: cacheCreate({ name: 'user:blocks', max: 100, - length: function () { return 1; }, - maxAge: 0, + ttl: 0, }), }; diff --git a/src/views/admin/advanced/cache.tpl b/src/views/admin/advanced/cache.tpl index 8de6734da0..3ac75a0d38 100644 --- a/src/views/admin/advanced/cache.tpl +++ b/src/views/admin/advanced/cache.tpl @@ -12,7 +12,7 @@ - {postCache.length} / {postCache.max}
+ {postCache.length} / {postCache.maxSize}
@@ -46,7 +46,7 @@
- {objectCache.length} / {objectCache.max}
+ {objectCache.itemCount} / {objectCache.max}
[[admin/advanced/cache:percent-full, {objectCache.percentFull}]] @@ -72,7 +72,7 @@
- {groupCache.length} / {groupCache.max}
+ {groupCache.itemCount} / {groupCache.max}
@@ -98,7 +98,7 @@
- {localCache.length} / {localCache.max}
+ {localCache.itemCount} / {localCache.max}
diff --git a/test/middleware.js b/test/middleware.js index cdb2797360..47567717d1 100644 --- a/test/middleware.js +++ b/test/middleware.js @@ -4,7 +4,7 @@ const assert = require('assert'); const nconf = require('nconf'); const request = require('request-promise-native'); const db = require('./mocks/databasemock'); -const middleware = require('../src/middleware'); + const user = require('../src/user'); const groups = require('../src/groups'); const utils = require('../src/utils'); @@ -21,6 +21,7 @@ describe('Middlewares', () => { }); it('should expose res.locals.isAdmin = false', (done) => { + const middleware = require('../src/middleware'); const resMock = { locals: {} }; middleware.exposeAdmin({}, resMock, () => { assert.strictEqual(resMock.locals.isAdmin, false); @@ -29,6 +30,7 @@ describe('Middlewares', () => { }); it('should expose res.locals.isAdmin = true', (done) => { + const middleware = require('../src/middleware'); const reqMock = { user: { uid: adminUid } }; const resMock = { locals: {} }; middleware.exposeAdmin(reqMock, resMock, () => { @@ -38,6 +40,7 @@ describe('Middlewares', () => { }); it('should expose privileges in res.locals.privileges and isSelf=true', (done) => { + const middleware = require('../src/middleware'); const reqMock = { user: { uid: adminUid }, params: { uid: adminUid } }; const resMock = { locals: {} }; middleware.exposePrivileges(reqMock, resMock, () => { @@ -51,6 +54,7 @@ describe('Middlewares', () => { }); it('should expose privileges in res.locals.privileges and isSelf=false', (done) => { + const middleware = require('../src/middleware'); const reqMock = { user: { uid: 0 }, params: { uid: adminUid } }; const resMock = { locals: {} }; middleware.exposePrivileges(reqMock, resMock, () => { @@ -64,6 +68,7 @@ describe('Middlewares', () => { }); it('should expose privilege set', (done) => { + const middleware = require('../src/middleware'); const reqMock = { user: { uid: adminUid } }; const resMock = { locals: {} }; middleware.exposePrivilegeSet(reqMock, resMock, () => { diff --git a/test/mocks/databasemock.js b/test/mocks/databasemock.js index d9fd9ac647..d22e4c01b0 100644 --- a/test/mocks/databasemock.js +++ b/test/mocks/databasemock.js @@ -184,20 +184,21 @@ async function setupMockDefaults() { const meta = require('../../src/meta'); await db.emptydb(); - require('../../src/groups').cache.reset(); - require('../../src/posts/cache').reset(); - require('../../src/cache').reset(); - require('../../src/middleware/uploads').clearCache(); - winston.info('test_database flushed'); await setupDefaultConfigs(meta); - await giveDefaultGlobalPrivileges(); + await meta.configs.init(); meta.config.postDelay = 0; meta.config.initialPostDelay = 0; meta.config.newbiePostDelay = 0; meta.config.autoDetectLang = 0; + require('../../src/groups').cache.reset(); + require('../../src/posts/cache').reset(); + require('../../src/cache').reset(); + require('../../src/middleware/uploads').clearCache(); + // privileges must be given after cache reset + await giveDefaultGlobalPrivileges(); await enableDefaultPlugins(); await meta.themes.set({ diff --git a/test/uploads.js b/test/uploads.js index 2d57ea4535..8e851d153d 100644 --- a/test/uploads.js +++ b/test/uploads.js @@ -22,7 +22,6 @@ const helpers = require('./helpers'); const file = require('../src/file'); const image = require('../src/image'); -const uploadFile = util.promisify(helpers.uploadFile); const emptyUploadsFolder = async () => { const files = await fs.readdir(`${nconf.get('upload_path')}/files`); await Promise.all(files.map(async (filename) => { @@ -517,7 +516,7 @@ describe('Upload Controllers', () => { describe('.getOrphans()', () => { before(async () => { const { jar, csrf_token } = await helpers.loginUser('regular', 'zugzug'); - await uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); + await helpers.uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); }); it('should return files with no post associated with them', async () => { @@ -538,7 +537,7 @@ describe('Upload Controllers', () => { before(async () => { const { jar, csrf_token } = await helpers.loginUser('regular', 'zugzug'); - await uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); + await helpers.uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); // modify all files in uploads folder to be 30 days old const files = await fs.readdir(`${nconf.get('upload_path')}/files`); From 64541dd820f04f476989f0ab16e8ac2860cf47ee Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 10 Aug 2022 13:22:25 -0400 Subject: [PATCH 02/11] Revert "Update to lru-cache@^7 (#10813)" This reverts commit c9754b09e67325ea44156b3675cc45c6fe890cda. --- install/package.json | 6 +-- src/analytics.js | 7 +-- src/cache.js | 2 +- src/cacheCreate.js | 74 +++++------------------------- src/controllers/admin/cache.js | 5 +- src/database/cache.js | 3 +- src/groups/cache.js | 2 +- src/middleware/index.js | 6 +-- src/middleware/uploads.js | 9 ++-- src/posts/cache.js | 6 +-- src/user/blocks.js | 3 +- src/views/admin/advanced/cache.tpl | 8 ++-- test/middleware.js | 7 +-- test/mocks/databasemock.js | 13 +++--- test/uploads.js | 5 +- 15 files changed, 50 insertions(+), 106 deletions(-) diff --git a/install/package.json b/install/package.json index d27a13a5ee..0a62a66e88 100644 --- a/install/package.json +++ b/install/package.json @@ -77,7 +77,7 @@ "less": "4.1.3", "lodash": "4.17.21", "logrotate-stream": "0.2.8", - "lru-cache": "7.13.1", + "lru-cache": "6.0.0", "material-design-lite": "1.3.0", "mime": "3.0.0", "mkdirp": "1.0.4", @@ -92,7 +92,7 @@ "nodebb-plugin-dbsearch": "5.1.5", "nodebb-plugin-emoji": "4.0.4", "nodebb-plugin-emoji-android": "3.0.0", - "nodebb-plugin-markdown": "10.1.0", + "nodebb-plugin-markdown": "10.0.0", "nodebb-plugin-mentions": "3.0.11", "nodebb-plugin-spam-be-gone": "1.0.0", "nodebb-rewards-essentials": "0.2.1", @@ -186,4 +186,4 @@ "url": "https://github.com/barisusakli" } ] -} +} \ No newline at end of file diff --git a/src/analytics.js b/src/analytics.js index f9e14b4caf..5d71d161a8 100644 --- a/src/analytics.js +++ b/src/analytics.js @@ -4,6 +4,7 @@ const cronJob = require('cron').CronJob; const winston = require('winston'); const nconf = require('nconf'); const crypto = require('crypto'); +const LRU = require('lru-cache'); const util = require('util'); const _ = require('lodash'); @@ -14,7 +15,6 @@ const utils = require('./utils'); const plugins = require('./plugins'); const meta = require('./meta'); const pubsub = require('./pubsub'); -const cacheCreate = require('./cacheCreate'); const Analytics = module.exports; @@ -37,9 +37,10 @@ let ipCache; const runJobs = nconf.get('runJobs'); Analytics.init = async function () { - ipCache = cacheCreate({ + ipCache = new LRU({ max: parseInt(meta.config['analytics:maxCache'], 10) || 500, - ttl: 0, + length: function () { return 1; }, + maxAge: 0, }); new cronJob('*/10 * * * * *', (async () => { diff --git a/src/cache.js b/src/cache.js index 6dcc440536..7bb27961ce 100644 --- a/src/cache.js +++ b/src/cache.js @@ -5,5 +5,5 @@ const cacheCreate = require('./cacheCreate'); module.exports = cacheCreate({ name: 'local', max: 40000, - ttl: 0, + maxAge: 0, }); diff --git a/src/cacheCreate.js b/src/cacheCreate.js index deaef9bba4..e3d7342165 100644 --- a/src/cacheCreate.js +++ b/src/cacheCreate.js @@ -4,72 +4,30 @@ module.exports = function (opts) { const LRU = require('lru-cache'); const pubsub = require('./pubsub'); - // lru-cache@7 deprecations - const winston = require('winston'); - const chalk = require('chalk'); + const cache = new LRU(opts); - // sometimes we kept passing in `length` with no corresponding `maxSize`. - // This is now enforced in v7; drop superfluous property - if (opts.hasOwnProperty('length') && !opts.hasOwnProperty('maxSize')) { - winston.warn(`[cache/init(${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} ${chalk.yellow('length')} was passed in without a corresponding ${chalk.yellow('maxSize')}. Both are now required as of lru-cache@7.0.0.`); - delete opts.length; - } - - const deprecations = new Map([ - ['stale', 'allowStale'], - ['maxAge', 'ttl'], - ['length', 'sizeCalculation'], - ]); - deprecations.forEach((newProp, oldProp) => { - if (opts.hasOwnProperty(oldProp) && !opts.hasOwnProperty(newProp)) { - winston.warn(`[cache/init (${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} The option ${chalk.yellow(oldProp)} has been deprecated as of lru-cache@7.0.0. Please change this to ${chalk.yellow(newProp)} instead.`); - opts[newProp] = opts[oldProp]; - delete opts[oldProp]; - } - }); - - const lruCache = new LRU(opts); - - const cache = {}; cache.name = opts.name; cache.hits = 0; cache.misses = 0; cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true; - const cacheSet = lruCache.set; - // backwards compatibility - const propertyMap = new Map([ - ['length', 'calculatedSize'], - ['max', 'max'], - ['maxSize', 'maxSize'], - ['itemCount', 'size'], - ]); - propertyMap.forEach((lruProp, cacheProp) => { - Object.defineProperty(cache, cacheProp, { - get: function () { - return lruCache[lruProp]; - }, - configurable: true, - enumerable: true, - }); - }); + const cacheSet = cache.set; + const cacheGet = cache.get; + const cacheDel = cache.del; + const cacheReset = cache.reset; - cache.set = function (key, value, ttl) { + cache.set = function (key, value, maxAge) { if (!cache.enabled) { return; } - const opts = {}; - if (ttl) { - opts.ttl = ttl; - } - cacheSet.apply(lruCache, [key, value, opts]); + cacheSet.apply(cache, [key, value, maxAge]); }; cache.get = function (key) { if (!cache.enabled) { return undefined; } - const data = lruCache.get(key); + const data = cacheGet.apply(cache, [key]); if (data === undefined) { cache.misses += 1; } else { @@ -83,18 +41,16 @@ module.exports = function (opts) { keys = [keys]; } pubsub.publish(`${cache.name}:cache:del`, keys); - keys.forEach(key => lruCache.delete(key)); + keys.forEach(key => cacheDel.apply(cache, [key])); }; - cache.delete = cache.del; cache.reset = function () { pubsub.publish(`${cache.name}:cache:reset`); localReset(); }; - cache.clear = cache.reset; function localReset() { - lruCache.clear(); + cacheReset.apply(cache); cache.hits = 0; cache.misses = 0; } @@ -105,7 +61,7 @@ module.exports = function (opts) { pubsub.on(`${cache.name}:cache:del`, (keys) => { if (Array.isArray(keys)) { - keys.forEach(key => lruCache.delete(key)); + keys.forEach(key => cacheDel.apply(cache, [key])); } }); @@ -131,13 +87,5 @@ module.exports = function (opts) { return unCachedKeys; }; - cache.dump = function () { - return lruCache.dump(); - }; - - cache.peek = function (key) { - return lruCache.peek(key); - }; - return cache; }; diff --git a/src/controllers/admin/cache.js b/src/controllers/admin/cache.js index 67c9864ae5..430cc28b8a 100644 --- a/src/controllers/admin/cache.js +++ b/src/controllers/admin/cache.js @@ -14,11 +14,8 @@ cacheController.get = function (req, res) { return { length: cache.length, max: cache.max, - maxSize: cache.maxSize, itemCount: cache.itemCount, - percentFull: cache.name === 'post' ? - ((cache.length / cache.maxSize) * 100).toFixed(2) : - ((cache.itemCount / cache.max) * 100).toFixed(2), + percentFull: ((cache.length / cache.max) * 100).toFixed(2), hits: utils.addCommas(String(cache.hits)), misses: utils.addCommas(String(cache.misses)), hitRatio: ((cache.hits / (cache.hits + cache.misses) || 0)).toFixed(4), diff --git a/src/database/cache.js b/src/database/cache.js index 999c860363..069b181a80 100644 --- a/src/database/cache.js +++ b/src/database/cache.js @@ -5,6 +5,7 @@ module.exports.create = function (name) { return cacheCreate({ name: `${name}-object`, max: 40000, - ttl: 0, + length: function () { return 1; }, + maxAge: 0, }); }; diff --git a/src/groups/cache.js b/src/groups/cache.js index b83e3e5928..0acea646d6 100644 --- a/src/groups/cache.js +++ b/src/groups/cache.js @@ -6,7 +6,7 @@ module.exports = function (Groups) { Groups.cache = cacheCreate({ name: 'group', max: 40000, - ttl: 0, + maxAge: 0, }); Groups.clearCache = function (uid, groupNames) { diff --git a/src/middleware/index.js b/src/middleware/index.js index 6930021aee..a31cc4430d 100644 --- a/src/middleware/index.js +++ b/src/middleware/index.js @@ -6,6 +6,7 @@ const csrf = require('csurf'); const validator = require('validator'); const nconf = require('nconf'); const toobusy = require('toobusy-js'); +const LRU = require('lru-cache'); const util = require('util'); const plugins = require('../plugins'); @@ -14,7 +15,6 @@ const user = require('../user'); const groups = require('../groups'); const analytics = require('../analytics'); const privileges = require('../privileges'); -const cacheCreate = require('../cacheCreate'); const helpers = require('./helpers'); const controllers = { @@ -22,8 +22,8 @@ const controllers = { helpers: require('../controllers/helpers'), }; -const delayCache = cacheCreate({ - ttl: 1000 * 60, +const delayCache = new LRU({ + maxAge: 1000 * 60, }); const middleware = module.exports; diff --git a/src/middleware/uploads.js b/src/middleware/uploads.js index beb36c03e0..941b9c27bc 100644 --- a/src/middleware/uploads.js +++ b/src/middleware/uploads.js @@ -1,16 +1,16 @@ 'use strict'; -const cacheCreate = require('../cacheCreate'); +const LRU = require('lru-cache'); const meta = require('../meta'); const helpers = require('./helpers'); const user = require('../user'); -const cache = cacheCreate({ - ttl: meta.config.uploadRateLimitCooldown * 1000, +const cache = new LRU({ + maxAge: meta.config.uploadRateLimitCooldown * 1000, }); exports.clearCache = function () { - cache.clear(); + cache.reset(); }; exports.ratelimit = helpers.try(async (req, res, next) => { @@ -23,6 +23,7 @@ exports.ratelimit = helpers.try(async (req, res, next) => { if (count > meta.config.uploadRateLimitThreshold) { return next(new Error(['[[error:upload-ratelimit-reached]]'])); } + cache.set(`${req.ip}:uploaded_file_count`, count); next(); }); diff --git a/src/posts/cache.js b/src/posts/cache.js index 246074697c..061c46d88d 100644 --- a/src/posts/cache.js +++ b/src/posts/cache.js @@ -5,8 +5,8 @@ const meta = require('../meta'); module.exports = cacheCreate({ name: 'post', - maxSize: meta.config.postCacheSize, - sizeCalculation: function (n) { return n.length; }, - ttl: 0, + max: meta.config.postCacheSize, + length: function (n) { return n.length; }, + maxAge: 0, enabled: global.env === 'production', }); diff --git a/src/user/blocks.js b/src/user/blocks.js index f71589911c..6655965f2a 100644 --- a/src/user/blocks.js +++ b/src/user/blocks.js @@ -9,7 +9,8 @@ module.exports = function (User) { _cache: cacheCreate({ name: 'user:blocks', max: 100, - ttl: 0, + length: function () { return 1; }, + maxAge: 0, }), }; diff --git a/src/views/admin/advanced/cache.tpl b/src/views/admin/advanced/cache.tpl index 3ac75a0d38..8de6734da0 100644 --- a/src/views/admin/advanced/cache.tpl +++ b/src/views/admin/advanced/cache.tpl @@ -12,7 +12,7 @@
- {postCache.length} / {postCache.maxSize}
+ {postCache.length} / {postCache.max}
@@ -46,7 +46,7 @@
- {objectCache.itemCount} / {objectCache.max}
+ {objectCache.length} / {objectCache.max}
[[admin/advanced/cache:percent-full, {objectCache.percentFull}]] @@ -72,7 +72,7 @@
- {groupCache.itemCount} / {groupCache.max}
+ {groupCache.length} / {groupCache.max}
@@ -98,7 +98,7 @@
- {localCache.itemCount} / {localCache.max}
+ {localCache.length} / {localCache.max}
diff --git a/test/middleware.js b/test/middleware.js index 47567717d1..cdb2797360 100644 --- a/test/middleware.js +++ b/test/middleware.js @@ -4,7 +4,7 @@ const assert = require('assert'); const nconf = require('nconf'); const request = require('request-promise-native'); const db = require('./mocks/databasemock'); - +const middleware = require('../src/middleware'); const user = require('../src/user'); const groups = require('../src/groups'); const utils = require('../src/utils'); @@ -21,7 +21,6 @@ describe('Middlewares', () => { }); it('should expose res.locals.isAdmin = false', (done) => { - const middleware = require('../src/middleware'); const resMock = { locals: {} }; middleware.exposeAdmin({}, resMock, () => { assert.strictEqual(resMock.locals.isAdmin, false); @@ -30,7 +29,6 @@ describe('Middlewares', () => { }); it('should expose res.locals.isAdmin = true', (done) => { - const middleware = require('../src/middleware'); const reqMock = { user: { uid: adminUid } }; const resMock = { locals: {} }; middleware.exposeAdmin(reqMock, resMock, () => { @@ -40,7 +38,6 @@ describe('Middlewares', () => { }); it('should expose privileges in res.locals.privileges and isSelf=true', (done) => { - const middleware = require('../src/middleware'); const reqMock = { user: { uid: adminUid }, params: { uid: adminUid } }; const resMock = { locals: {} }; middleware.exposePrivileges(reqMock, resMock, () => { @@ -54,7 +51,6 @@ describe('Middlewares', () => { }); it('should expose privileges in res.locals.privileges and isSelf=false', (done) => { - const middleware = require('../src/middleware'); const reqMock = { user: { uid: 0 }, params: { uid: adminUid } }; const resMock = { locals: {} }; middleware.exposePrivileges(reqMock, resMock, () => { @@ -68,7 +64,6 @@ describe('Middlewares', () => { }); it('should expose privilege set', (done) => { - const middleware = require('../src/middleware'); const reqMock = { user: { uid: adminUid } }; const resMock = { locals: {} }; middleware.exposePrivilegeSet(reqMock, resMock, () => { diff --git a/test/mocks/databasemock.js b/test/mocks/databasemock.js index d22e4c01b0..d9fd9ac647 100644 --- a/test/mocks/databasemock.js +++ b/test/mocks/databasemock.js @@ -184,21 +184,20 @@ async function setupMockDefaults() { const meta = require('../../src/meta'); await db.emptydb(); + require('../../src/groups').cache.reset(); + require('../../src/posts/cache').reset(); + require('../../src/cache').reset(); + require('../../src/middleware/uploads').clearCache(); + winston.info('test_database flushed'); await setupDefaultConfigs(meta); - + await giveDefaultGlobalPrivileges(); await meta.configs.init(); meta.config.postDelay = 0; meta.config.initialPostDelay = 0; meta.config.newbiePostDelay = 0; meta.config.autoDetectLang = 0; - require('../../src/groups').cache.reset(); - require('../../src/posts/cache').reset(); - require('../../src/cache').reset(); - require('../../src/middleware/uploads').clearCache(); - // privileges must be given after cache reset - await giveDefaultGlobalPrivileges(); await enableDefaultPlugins(); await meta.themes.set({ diff --git a/test/uploads.js b/test/uploads.js index 8e851d153d..2d57ea4535 100644 --- a/test/uploads.js +++ b/test/uploads.js @@ -22,6 +22,7 @@ const helpers = require('./helpers'); const file = require('../src/file'); const image = require('../src/image'); +const uploadFile = util.promisify(helpers.uploadFile); const emptyUploadsFolder = async () => { const files = await fs.readdir(`${nconf.get('upload_path')}/files`); await Promise.all(files.map(async (filename) => { @@ -516,7 +517,7 @@ describe('Upload Controllers', () => { describe('.getOrphans()', () => { before(async () => { const { jar, csrf_token } = await helpers.loginUser('regular', 'zugzug'); - await helpers.uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); + await uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); }); it('should return files with no post associated with them', async () => { @@ -537,7 +538,7 @@ describe('Upload Controllers', () => { before(async () => { const { jar, csrf_token } = await helpers.loginUser('regular', 'zugzug'); - await helpers.uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); + await uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); // modify all files in uploads folder to be 30 days old const files = await fs.readdir(`${nconf.get('upload_path')}/files`); From 55254422794f9d221a996d306c5ab44c33989057 Mon Sep 17 00:00:00 2001 From: Misty Release Bot Date: Wed, 10 Aug 2022 20:02:18 +0000 Subject: [PATCH 03/11] chore: incrementing version number - v2.4.0 --- install/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/package.json b/install/package.json index 94c22d2872..ff4b2c4814 100644 --- a/install/package.json +++ b/install/package.json @@ -2,7 +2,7 @@ "name": "nodebb", "license": "GPL-3.0", "description": "NodeBB Forum", - "version": "2.3.1", + "version": "2.4.0", "homepage": "http://www.nodebb.org", "repository": { "type": "git", @@ -29,7 +29,7 @@ }, "dependencies": { "@adactive/bootstrap-tagsinput": "0.8.2", - "@isaacs/ttlcache": "^1.2.0", + "@isaacs/ttlcache": "^1.2.0", "ace-builds": "1.8.1", "archiver": "5.3.1", "async": "3.2.4", @@ -187,4 +187,4 @@ "url": "https://github.com/barisusakli" } ] -} +} \ No newline at end of file From c4714ff7df9cc48b8fb157c5232ea002790d27ac Mon Sep 17 00:00:00 2001 From: Misty Release Bot Date: Wed, 10 Aug 2022 20:02:19 +0000 Subject: [PATCH 04/11] chore: update changelog for v2.4.0 --- CHANGELOG.md | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index addb024ffd..57f2822f2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,67 @@ +#### v2.4.0 (2022-08-10) + +##### Chores + +* **deps:** + * update dependency eslint to v8.21.0 (13a17bd1) + * bump commander from 7.2.0 to 9.4.0 in /install (993b7747) +* update to new transifex project url (659cfe85) +* re-order interstitial tests so email and gdpr tests are in sub-blocks (342cca35) +* opt-out of dependabot, due to conflicts with renovate (70d60289) +* incrementing version number - v2.3.1 (d2425942) +* update changelog for v2.3.1 (2f487175) +* incrementing version number - v2.3.0 (046ea120) +* **i18n:** + * fallback strings for new resources: nodebb.admin-settings-email (cdaa8f21) + * fallback strings for new resources: nodebb.admin-settings-email (3e56c547) + * fallback strings for new resources: nodebb.user (bcf7ef67) + +##### New Features + +* support packageManager property in package.json (b3a37a7f) +* automatically enable the SMTP transport option if the SMTP service is changed (4055e3bd) +* present a password challenge on email update flow (7fcee42b) +* add client side filter:chat.send, closes #10729 (cb084cbd) +* fire hook to allow plugins to filter the pids returned in a user profile (17e44ff5) +* closes #10719, don't trim children if category is marked section (be917e8d) +* closes #10719, don't trim children if category is marked section (0bec52bc) + +##### Bug Fixes + +* adapt to breaking change in commander (38bf30c8) +* move panel-offset setting code back to theme header (d0255fc6) +* #10808; tweak copy for gmail app passwords support (7082291b) +* don't require password challenge if no password is set in user account (9d27e907) +* do not throw if password passed into `isPasswordCorrect` is invalid, just return false (287f4c2c) +* don't crash if req.body.username is not string (7e8ad785) +* don't crash if target/user is undefined (55c5588a) +* race condition causing undefined ajaxify.data (4586f68e) +* #10809, test runner to only run tests for plugins included in `test_plugins` (1ca09b63) +* #10805, hide unconfirmed emails from user data retrieval methods (cba9047f) +* use different emoji on NodeBB Ready — again because procrastination (3e062a7f) +* unnecessary escape (cd438b32) +* remove socket.io cluster adapter (#10742) (456b8798) +* #10783, do not purge files without a timestamp prefix (dc3a6a29) +* **deps:** + * bump persona v12.1.0 (1465598d) + * bump 2factor to v5.0.2 (bd18004d) + * update dependency sanitize-html to v2.7.1 (#10792) (f02492bd) + * update dependency html-to-text to v8.2.1 (f22790c0) + * update dependency webpack to v5.74.0 (e748e31f) + * update dependency autoprefixer to v10.4.8 (#10799) (4ca0d571) + +##### Performance Improvements + +* make single db call (d73f0f9c) + +##### Tests + +* additional tests for password challenge on email update (65c59cc1) +* add dummy emailer hook to suppress sendmail error logging (8e1a4bb5) +* fix one last failing test (68bcd7f4) +* fix user email tests (06f089af) +* fix tests so that when user.create is called, email is set prior to confirmation (f93a0b83) + #### v2.3.1 (2022-07-29) ##### Chores From 371ac03220426db95b0009d29eb2e93b3aa6fae1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 19:33:09 +0000 Subject: [PATCH 05/11] chore(deps): update docker/setup-buildx-action action to v2 --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 86dc5c50d3..e9573c3ebb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,7 +24,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub uses: docker/login-action@v1 From 3d68accf993e0c0a6d7928c9d4b3367e504e8c38 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 15:04:57 +0000 Subject: [PATCH 06/11] chore(deps): update docker/login-action action to v2 --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e9573c3ebb..bd803a808d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,7 +27,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From bfd6318cd626ba71edf47cf2a12f225a80d4563c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 15:03:25 +0000 Subject: [PATCH 07/11] chore(deps): update docker/build-push-action action to v3 --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bd803a808d..eab9315e0a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,7 +43,7 @@ jobs: type=raw,value=latest - name: Build and push Docker images - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . file: ./Dockerfile From d2c2f333e13e6ef57ad87b749652f2e702534a0e Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 12 Aug 2022 17:14:20 -0400 Subject: [PATCH 08/11] Revert "chore(deps): update docker/build-push-action action to v3" This reverts commit bfd6318cd626ba71edf47cf2a12f225a80d4563c. --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eab9315e0a..bd803a808d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,7 +43,7 @@ jobs: type=raw,value=latest - name: Build and push Docker images - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: context: . file: ./Dockerfile From 887df0e62b1c31c9b2d2b8228a0cba1e591c7638 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 12 Aug 2022 17:14:21 -0400 Subject: [PATCH 09/11] Revert "chore(deps): update docker/login-action action to v2" This reverts commit 3d68accf993e0c0a6d7928c9d4b3367e504e8c38. --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bd803a808d..e9573c3ebb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,7 +27,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From 8936f412d16c824bc247b393622a499fc5161617 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 12 Aug 2022 17:14:22 -0400 Subject: [PATCH 10/11] Revert "chore(deps): update docker/setup-buildx-action action to v2" This reverts commit 371ac03220426db95b0009d29eb2e93b3aa6fae1. --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e9573c3ebb..86dc5c50d3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,7 +24,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 - name: Login to Docker Hub uses: docker/login-action@v1 From 15ca460c8f144c3167249b135902ac59289ca2f8 Mon Sep 17 00:00:00 2001 From: Opliko Date: Sat, 13 Aug 2022 15:29:56 +0200 Subject: [PATCH 11/11] fix: return at least one in sizeCalculation (#10832) if post content is empty post cache should still consider its size to be at least one. fixes #10831 --- src/posts/cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/posts/cache.js b/src/posts/cache.js index 888fba6424..7f4711d0cd 100644 --- a/src/posts/cache.js +++ b/src/posts/cache.js @@ -6,7 +6,7 @@ const meta = require('../meta'); module.exports = cacheCreate({ name: 'post', maxSize: meta.config.postCacheSize, - sizeCalculation: function (n) { return n.length; }, + sizeCalculation: function (n) { return n.length || 1; }, ttl: 0, enabled: global.env === 'production', });