hyperzlib
/
nodebb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "Update to lru-cache@^7 (#10813)"

Browse Source 
This reverts commit c9754b09e6.
isekai-main
Julian Lam 3 years ago
parent c9754b09e6
commit 64541dd820
15 changed files with 50 additions and 106 deletions
Show Stats Download Patch File Download Diff File

6
install/package.json
Unescape Escape View File

@ -77,7 +77,7 @@
"less": "4.1.3", "less": "4.1.3",
"lodash": "4.17.21", "lodash": "4.17.21",
"logrotate-stream": "0.2.8", "logrotate-stream": "0.2.8",
"lru-cache": "7.13.1", "lru-cache": "6.0.0",
"material-design-lite": "1.3.0", "material-design-lite": "1.3.0",
"mime": "3.0.0", "mime": "3.0.0",
"mkdirp": "1.0.4", "mkdirp": "1.0.4",
@ -92,7 +92,7 @@
"nodebb-plugin-dbsearch": "5.1.5", "nodebb-plugin-dbsearch": "5.1.5",
"nodebb-plugin-emoji": "4.0.4", "nodebb-plugin-emoji": "4.0.4",
"nodebb-plugin-emoji-android": "3.0.0", "nodebb-plugin-emoji-android": "3.0.0",
"nodebb-plugin-markdown": "10.1.0", "nodebb-plugin-markdown": "10.0.0",
"nodebb-plugin-mentions": "3.0.11", "nodebb-plugin-mentions": "3.0.11",
"nodebb-plugin-spam-be-gone": "1.0.0", "nodebb-plugin-spam-be-gone": "1.0.0",
"nodebb-rewards-essentials": "0.2.1", "nodebb-rewards-essentials": "0.2.1",
@ -186,4 +186,4 @@
"url": "https://github.com/barisusakli" "url": "https://github.com/barisusakli"
} }
] ]
} }

7
src/analytics.js
Unescape Escape View File

@ -4,6 +4,7 @@ const cronJob = require('cron').CronJob;
const winston = require('winston'); const winston = require('winston');
const nconf = require('nconf'); const nconf = require('nconf');
const crypto = require('crypto'); const crypto = require('crypto');
const LRU = require('lru-cache');
const util = require('util'); const util = require('util');
const _ = require('lodash'); const _ = require('lodash');
@ -14,7 +15,6 @@ const utils = require('./utils');
const plugins = require('./plugins'); const plugins = require('./plugins');
const meta = require('./meta'); const meta = require('./meta');
const pubsub = require('./pubsub'); const pubsub = require('./pubsub');
const cacheCreate = require('./cacheCreate');
const Analytics = module.exports; const Analytics = module.exports;
@ -37,9 +37,10 @@ let ipCache;
const runJobs = nconf.get('runJobs'); const runJobs = nconf.get('runJobs');
Analytics.init = async function () { Analytics.init = async function () {
ipCache = cacheCreate({ ipCache = new LRU({
max: parseInt(meta.config['analytics:maxCache'], 10) || 500, max: parseInt(meta.config['analytics:maxCache'], 10) || 500,
ttl: 0, length: function () { return 1; },
maxAge: 0,
}); });
new cronJob('*/10 * * * * *', (async () => { new cronJob('*/10 * * * * *', (async () => {

2
src/cache.js
Unescape Escape View File

@ -5,5 +5,5 @@ const cacheCreate = require('./cacheCreate');
module.exports = cacheCreate({ module.exports = cacheCreate({
name: 'local', name: 'local',
max: 40000, max: 40000,
ttl: 0, maxAge: 0,
}); });

74
src/cacheCreate.js
Unescape Escape View File

@ -4,72 +4,30 @@ module.exports = function (opts) {
const LRU = require('lru-cache'); const LRU = require('lru-cache');
const pubsub = require('./pubsub'); const pubsub = require('./pubsub');
// lru-cache@7 deprecations const cache = new LRU(opts);
const winston = require('winston');
const chalk = require('chalk');
// sometimes we kept passing in `length` with no corresponding `maxSize`.
// This is now enforced in v7; drop superfluous property
if (opts.hasOwnProperty('length') && !opts.hasOwnProperty('maxSize')) {
winston.warn(`[cache/init(${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} ${chalk.yellow('length')} was passed in without a corresponding ${chalk.yellow('maxSize')}. Both are now required as of lru-cache@7.0.0.`);
delete opts.length;
}
const deprecations = new Map([
['stale', 'allowStale'],
['maxAge', 'ttl'],
['length', 'sizeCalculation'],
]);
deprecations.forEach((newProp, oldProp) => {
if (opts.hasOwnProperty(oldProp) && !opts.hasOwnProperty(newProp)) {
winston.warn(`[cache/init (${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} The option ${chalk.yellow(oldProp)} has been deprecated as of lru-cache@7.0.0. Please change this to ${chalk.yellow(newProp)} instead.`);
opts[newProp] = opts[oldProp];
delete opts[oldProp];
}
});
const lruCache = new LRU(opts);
const cache = {};
cache.name = opts.name; cache.name = opts.name;
cache.hits = 0; cache.hits = 0;
cache.misses = 0; cache.misses = 0;
cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true; cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true;
const cacheSet = lruCache.set;
// backwards compatibility const cacheSet = cache.set;
const propertyMap = new Map([ const cacheGet = cache.get;
['length', 'calculatedSize'], const cacheDel = cache.del;
['max', 'max'], const cacheReset = cache.reset;
['maxSize', 'maxSize'],
['itemCount', 'size'],
]);
propertyMap.forEach((lruProp, cacheProp) => {
Object.defineProperty(cache, cacheProp, {
get: function () {
return lruCache[lruProp];
},
configurable: true,
enumerable: true,
});
});
cache.set = function (key, value, ttl) { cache.set = function (key, value, maxAge) {
if (!cache.enabled) { if (!cache.enabled) {
return; return;
} }
const opts = {}; cacheSet.apply(cache, [key, value, maxAge]);
if (ttl) {
opts.ttl = ttl;
}
cacheSet.apply(lruCache, [key, value, opts]);
}; };
cache.get = function (key) { cache.get = function (key) {
if (!cache.enabled) { if (!cache.enabled) {
return undefined; return undefined;
} }
const data = lruCache.get(key); const data = cacheGet.apply(cache, [key]);
if (data === undefined) { if (data === undefined) {
cache.misses += 1; cache.misses += 1;
} else { } else {
@ -83,18 +41,16 @@ module.exports = function (opts) {
keys = [keys]; keys = [keys];
} }
pubsub.publish(`${cache.name}:cache:del`, keys); pubsub.publish(`${cache.name}:cache:del`, keys);
keys.forEach(key => lruCache.delete(key)); keys.forEach(key => cacheDel.apply(cache, [key]));
}; };
cache.delete = cache.del;
cache.reset = function () { cache.reset = function () {
pubsub.publish(`${cache.name}:cache:reset`); pubsub.publish(`${cache.name}:cache:reset`);
localReset(); localReset();
}; };
cache.clear = cache.reset;
function localReset() { function localReset() {
lruCache.clear(); cacheReset.apply(cache);
cache.hits = 0; cache.hits = 0;
cache.misses = 0; cache.misses = 0;
} }
@ -105,7 +61,7 @@ module.exports = function (opts) {
pubsub.on(`${cache.name}:cache:del`, (keys) => { pubsub.on(`${cache.name}:cache:del`, (keys) => {
if (Array.isArray(keys)) { if (Array.isArray(keys)) {
keys.forEach(key => lruCache.delete(key)); keys.forEach(key => cacheDel.apply(cache, [key]));
} }
}); });
@ -131,13 +87,5 @@ module.exports = function (opts) {
return unCachedKeys; return unCachedKeys;
}; };
cache.dump = function () {
return lruCache.dump();
};
cache.peek = function (key) {
return lruCache.peek(key);
};
return cache; return cache;
}; };

5
src/controllers/admin/cache.js
Unescape Escape View File

@ -14,11 +14,8 @@ cacheController.get = function (req, res) {
return { return {
length: cache.length, length: cache.length,
max: cache.max, max: cache.max,
maxSize: cache.maxSize,
itemCount: cache.itemCount, itemCount: cache.itemCount,
percentFull: cache.name === 'post' ? percentFull: ((cache.length / cache.max) * 100).toFixed(2),
((cache.length / cache.maxSize) * 100).toFixed(2) :
((cache.itemCount / cache.max) * 100).toFixed(2),
hits: utils.addCommas(String(cache.hits)), hits: utils.addCommas(String(cache.hits)),
misses: utils.addCommas(String(cache.misses)), misses: utils.addCommas(String(cache.misses)),
hitRatio: ((cache.hits / (cache.hits + cache.misses) || 0)).toFixed(4), hitRatio: ((cache.hits / (cache.hits + cache.misses) || 0)).toFixed(4),

3
src/database/cache.js
Unescape Escape View File

@ -5,6 +5,7 @@ module.exports.create = function (name) {
return cacheCreate({ return cacheCreate({
name: `${name}-object`, name: `${name}-object`,
max: 40000, max: 40000,
ttl: 0, length: function () { return 1; },
maxAge: 0,
}); });
}; };

2
src/groups/cache.js
Unescape Escape View File

@ -6,7 +6,7 @@ module.exports = function (Groups) {
Groups.cache = cacheCreate({ Groups.cache = cacheCreate({
name: 'group', name: 'group',
max: 40000, max: 40000,
ttl: 0, maxAge: 0,
}); });
Groups.clearCache = function (uid, groupNames) { Groups.clearCache = function (uid, groupNames) {

6
src/middleware/index.js
Unescape Escape View File

@ -6,6 +6,7 @@ const csrf = require('csurf');
const validator = require('validator'); const validator = require('validator');
const nconf = require('nconf'); const nconf = require('nconf');
const toobusy = require('toobusy-js'); const toobusy = require('toobusy-js');
const LRU = require('lru-cache');
const util = require('util'); const util = require('util');
const plugins = require('../plugins'); const plugins = require('../plugins');
@ -14,7 +15,6 @@ const user = require('../user');
const groups = require('../groups'); const groups = require('../groups');
const analytics = require('../analytics'); const analytics = require('../analytics');
const privileges = require('../privileges'); const privileges = require('../privileges');
const cacheCreate = require('../cacheCreate');
const helpers = require('./helpers'); const helpers = require('./helpers');
const controllers = { const controllers = {
@ -22,8 +22,8 @@ const controllers = {
helpers: require('../controllers/helpers'), helpers: require('../controllers/helpers'),
}; };
const delayCache = cacheCreate({ const delayCache = new LRU({
ttl: 1000 * 60, maxAge: 1000 * 60,
}); });
const middleware = module.exports; const middleware = module.exports;

9
src/middleware/uploads.js
Unescape Escape View File

@ -1,16 +1,16 @@
'use strict'; 'use strict';
const cacheCreate = require('../cacheCreate'); const LRU = require('lru-cache');
const meta = require('../meta'); const meta = require('../meta');
const helpers = require('./helpers'); const helpers = require('./helpers');
const user = require('../user'); const user = require('../user');
const cache = cacheCreate({ const cache = new LRU({
ttl: meta.config.uploadRateLimitCooldown * 1000, maxAge: meta.config.uploadRateLimitCooldown * 1000,
}); });
exports.clearCache = function () { exports.clearCache = function () {
cache.clear(); cache.reset();
}; };
exports.ratelimit = helpers.try(async (req, res, next) => { exports.ratelimit = helpers.try(async (req, res, next) => {
@ -23,6 +23,7 @@ exports.ratelimit = helpers.try(async (req, res, next) => {
if (count > meta.config.uploadRateLimitThreshold) { if (count > meta.config.uploadRateLimitThreshold) {
return next(new Error(['[[error:upload-ratelimit-reached]]'])); return next(new Error(['[[error:upload-ratelimit-reached]]']));
} }
cache.set(`${req.ip}:uploaded_file_count`, count); cache.set(`${req.ip}:uploaded_file_count`, count);
next(); next();
}); });

6
src/posts/cache.js
Unescape Escape View File

@ -5,8 +5,8 @@ const meta = require('../meta');
module.exports = cacheCreate({ module.exports = cacheCreate({
name: 'post', name: 'post',
maxSize: meta.config.postCacheSize, max: meta.config.postCacheSize,
sizeCalculation: function (n) { return n.length; }, length: function (n) { return n.length; },
ttl: 0, maxAge: 0,
enabled: global.env === 'production', enabled: global.env === 'production',
}); });

3
src/user/blocks.js
Unescape Escape View File

@ -9,7 +9,8 @@ module.exports = function (User) {
_cache: cacheCreate({ _cache: cacheCreate({
name: 'user:blocks', name: 'user:blocks',
max: 100, max: 100,
ttl: 0, length: function () { return 1; },
maxAge: 0,
}), }),
}; };

8
src/views/admin/advanced/cache.tpl
Unescape Escape View File

@ -12,7 +12,7 @@
</label> </label>
</div> </div>
<span>{postCache.length} / {postCache.maxSize}</span><br/> <span>{postCache.length} / {postCache.max}</span><br/>
<div class="progress"> <div class="progress">
<div class="progress-bar" role="progressbar" aria-valuenow="{postCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {postCache.percentFull}%;"> <div class="progress-bar" role="progressbar" aria-valuenow="{postCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {postCache.percentFull}%;">
@ -46,7 +46,7 @@
<input class="mdl-switch__input" type="checkbox" {{{if objectCache.enabled}}}checked{{{end}}}> <input class="mdl-switch__input" type="checkbox" {{{if objectCache.enabled}}}checked{{{end}}}>
</label> </label>
</div> </div>
<span>{objectCache.itemCount} / {objectCache.max}</span><br/> <span>{objectCache.length} / {objectCache.max}</span><br/>
<div class="progress"> <div class="progress">
<div class="progress-bar" role="progressbar" aria-valuenow="{objectCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {objectCache.percentFull}%;"> <div class="progress-bar" role="progressbar" aria-valuenow="{objectCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {objectCache.percentFull}%;">
[[admin/advanced/cache:percent-full, {objectCache.percentFull}]] [[admin/advanced/cache:percent-full, {objectCache.percentFull}]]
@ -72,7 +72,7 @@
<input class="mdl-switch__input" type="checkbox" {{{if groupCache.enabled}}}checked{{{end}}}> <input class="mdl-switch__input" type="checkbox" {{{if groupCache.enabled}}}checked{{{end}}}>
</label> </label>
</div> </div>
<span>{groupCache.itemCount} / {groupCache.max}</span><br/> <span>{groupCache.length} / {groupCache.max}</span><br/>
<div class="progress"> <div class="progress">
<div class="progress-bar" role="progressbar" aria-valuenow="{groupCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {groupCache.percentFull}%;"> <div class="progress-bar" role="progressbar" aria-valuenow="{groupCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {groupCache.percentFull}%;">
@ -98,7 +98,7 @@
<input class="mdl-switch__input" type="checkbox" {{{if localCache.enabled}}}checked{{{end}}}> <input class="mdl-switch__input" type="checkbox" {{{if localCache.enabled}}}checked{{{end}}}>
</label> </label>
</div> </div>
<span>{localCache.itemCount} / {localCache.max}</span><br/> <span>{localCache.length} / {localCache.max}</span><br/>
<div class="progress"> <div class="progress">
<div class="progress-bar" role="progressbar" aria-valuenow="{localCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {localCache.percentFull}%;"> <div class="progress-bar" role="progressbar" aria-valuenow="{localCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {localCache.percentFull}%;">

7
test/middleware.js
Unescape Escape View File

@ -4,7 +4,7 @@ const assert = require('assert');
const nconf = require('nconf'); const nconf = require('nconf');
const request = require('request-promise-native'); const request = require('request-promise-native');
const db = require('./mocks/databasemock'); const db = require('./mocks/databasemock');
const middleware = require('../src/middleware');
const user = require('../src/user'); const user = require('../src/user');
const groups = require('../src/groups'); const groups = require('../src/groups');
const utils = require('../src/utils'); const utils = require('../src/utils');
@ -21,7 +21,6 @@ describe('Middlewares', () => {
}); });
it('should expose res.locals.isAdmin = false', (done) => { it('should expose res.locals.isAdmin = false', (done) => {
const middleware = require('../src/middleware');
const resMock = { locals: {} }; const resMock = { locals: {} };
middleware.exposeAdmin({}, resMock, () => { middleware.exposeAdmin({}, resMock, () => {
assert.strictEqual(resMock.locals.isAdmin, false); assert.strictEqual(resMock.locals.isAdmin, false);
@ -30,7 +29,6 @@ describe('Middlewares', () => {
}); });
it('should expose res.locals.isAdmin = true', (done) => { it('should expose res.locals.isAdmin = true', (done) => {
const middleware = require('../src/middleware');
const reqMock = { user: { uid: adminUid } }; const reqMock = { user: { uid: adminUid } };
const resMock = { locals: {} }; const resMock = { locals: {} };
middleware.exposeAdmin(reqMock, resMock, () => { middleware.exposeAdmin(reqMock, resMock, () => {
@ -40,7 +38,6 @@ describe('Middlewares', () => {
}); });
it('should expose privileges in res.locals.privileges and isSelf=true', (done) => { it('should expose privileges in res.locals.privileges and isSelf=true', (done) => {
const middleware = require('../src/middleware');
const reqMock = { user: { uid: adminUid }, params: { uid: adminUid } }; const reqMock = { user: { uid: adminUid }, params: { uid: adminUid } };
const resMock = { locals: {} }; const resMock = { locals: {} };
middleware.exposePrivileges(reqMock, resMock, () => { middleware.exposePrivileges(reqMock, resMock, () => {
@ -54,7 +51,6 @@ describe('Middlewares', () => {
}); });
it('should expose privileges in res.locals.privileges and isSelf=false', (done) => { it('should expose privileges in res.locals.privileges and isSelf=false', (done) => {
const middleware = require('../src/middleware');
const reqMock = { user: { uid: 0 }, params: { uid: adminUid } }; const reqMock = { user: { uid: 0 }, params: { uid: adminUid } };
const resMock = { locals: {} }; const resMock = { locals: {} };
middleware.exposePrivileges(reqMock, resMock, () => { middleware.exposePrivileges(reqMock, resMock, () => {
@ -68,7 +64,6 @@ describe('Middlewares', () => {
}); });
it('should expose privilege set', (done) => { it('should expose privilege set', (done) => {
const middleware = require('../src/middleware');
const reqMock = { user: { uid: adminUid } }; const reqMock = { user: { uid: adminUid } };
const resMock = { locals: {} }; const resMock = { locals: {} };
middleware.exposePrivilegeSet(reqMock, resMock, () => { middleware.exposePrivilegeSet(reqMock, resMock, () => {

13
test/mocks/databasemock.js
Unescape Escape View File

@ -184,21 +184,20 @@ async function setupMockDefaults() {
const meta = require('../../src/meta'); const meta = require('../../src/meta');
await db.emptydb(); await db.emptydb();
require('../../src/groups').cache.reset();
require('../../src/posts/cache').reset();
require('../../src/cache').reset();
require('../../src/middleware/uploads').clearCache();
winston.info('test_database flushed'); winston.info('test_database flushed');
await setupDefaultConfigs(meta); await setupDefaultConfigs(meta);
await giveDefaultGlobalPrivileges();
await meta.configs.init(); await meta.configs.init();
meta.config.postDelay = 0; meta.config.postDelay = 0;
meta.config.initialPostDelay = 0; meta.config.initialPostDelay = 0;
meta.config.newbiePostDelay = 0; meta.config.newbiePostDelay = 0;
meta.config.autoDetectLang = 0; meta.config.autoDetectLang = 0;
require('../../src/groups').cache.reset();
require('../../src/posts/cache').reset();
require('../../src/cache').reset();
require('../../src/middleware/uploads').clearCache();
// privileges must be given after cache reset
await giveDefaultGlobalPrivileges();
await enableDefaultPlugins(); await enableDefaultPlugins();
await meta.themes.set({ await meta.themes.set({

5
test/uploads.js
Unescape Escape View File

@ -22,6 +22,7 @@ const helpers = require('./helpers');
const file = require('../src/file'); const file = require('../src/file');
const image = require('../src/image'); const image = require('../src/image');
const uploadFile = util.promisify(helpers.uploadFile);
const emptyUploadsFolder = async () => { const emptyUploadsFolder = async () => {
const files = await fs.readdir(`${nconf.get('upload_path')}/files`); const files = await fs.readdir(`${nconf.get('upload_path')}/files`);
await Promise.all(files.map(async (filename) => { await Promise.all(files.map(async (filename) => {
@ -516,7 +517,7 @@ describe('Upload Controllers', () => {
describe('.getOrphans()', () => { describe('.getOrphans()', () => {
before(async () => { before(async () => {
const { jar, csrf_token } = await helpers.loginUser('regular', 'zugzug'); const { jar, csrf_token } = await helpers.loginUser('regular', 'zugzug');
await helpers.uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); await uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token);
}); });
it('should return files with no post associated with them', async () => { it('should return files with no post associated with them', async () => {
@ -537,7 +538,7 @@ describe('Upload Controllers', () => {
before(async () => { before(async () => {
const { jar, csrf_token } = await helpers.loginUser('regular', 'zugzug'); const { jar, csrf_token } = await helpers.loginUser('regular', 'zugzug');
await helpers.uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token); await uploadFile(`${nconf.get('url')}/api/post/upload`, path.join(__dirname, '../test/files/test.png'), {}, jar, csrf_token);
// modify all files in uploads folder to be 30 days old // modify all files in uploads folder to be 30 days old
const files = await fs.readdir(`${nconf.get('upload_path')}/files`); const files = await fs.readdir(`${nconf.get('upload_path')}/files`);

Write Preview
Loading…
Cancel
Save