hyperzlib
/
nodebb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TTL Cache (#10816)

Browse Source 
* refactor: move src/cacheCreate.js to src/cache/lru.js

* fix: call new library location for lru cache creator

* feat: add ttl cache

* fix: update upload throttler to use ttl cache instead of lru cache

* chore: add missing dependency

* fix: avoid pubsub conflicts

* fix: use get instead of peek, which is not available in ttl-cache
isekai-main
Julian Lam 2 years ago committed by GitHub
parent c07d595662
commit 9b753d6d57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 248 additions and 150 deletions
Show Stats Download Patch File Download Diff File

1
install/package.json
Unescape Escape View File

@ -29,6 +29,7 @@
},
"dependencies": {
"@adactive/bootstrap-tagsinput": "0.8.2",
"@isaacs/ttlcache": "^1.2.0",
"ace-builds": "1.8.1",
"archiver": "5.3.1",
"async": "3.2.4",

2
src/analytics.js
Unescape Escape View File

@ -14,7 +14,7 @@ const utils = require('./utils');
const plugins = require('./plugins');
const meta = require('./meta');
const pubsub = require('./pubsub');
const cacheCreate = require('./cacheCreate');
const cacheCreate = require('./cache/lru');
const Analytics = module.exports;

2
src/cache.js
Unescape Escape View File

@ -1,6 +1,6 @@
'use strict';
const cacheCreate = require('./cacheCreate');
const cacheCreate = require('./cache/lru');
module.exports = cacheCreate({
name: 'local',

143
src/cache/lru.js vendored
Unescape Escape View File

@ -0,0 +1,143 @@
'use strict';
module.exports = function (opts) {
const LRU = require('lru-cache');
const pubsub = require('../pubsub');
// lru-cache@7 deprecations
const winston = require('winston');
const chalk = require('chalk');
// sometimes we kept passing in `length` with no corresponding `maxSize`.
// This is now enforced in v7; drop superfluous property
if (opts.hasOwnProperty('length') && !opts.hasOwnProperty('maxSize')) {
winston.warn(`[cache/init(${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} ${chalk.yellow('length')} was passed in without a corresponding ${chalk.yellow('maxSize')}. Both are now required as of lru-cache@7.0.0.`);
delete opts.length;
}
const deprecations = new Map([
['stale', 'allowStale'],
['maxAge', 'ttl'],
['length', 'sizeCalculation'],
]);
deprecations.forEach((newProp, oldProp) => {
if (opts.hasOwnProperty(oldProp) && !opts.hasOwnProperty(newProp)) {
winston.warn(`[cache/init(${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} The option ${chalk.yellow(oldProp)} has been deprecated as of lru-cache@7.0.0. Please change this to ${chalk.yellow(newProp)} instead.`);
opts[newProp] = opts[oldProp];
delete opts[oldProp];
}
});
const lruCache = new LRU(opts);
const cache = {};
cache.name = opts.name;
cache.hits = 0;
cache.misses = 0;
cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true;
const cacheSet = lruCache.set;
// backwards compatibility
const propertyMap = new Map([
['length', 'calculatedSize'],
['max', 'max'],
['maxSize', 'maxSize'],
['itemCount', 'size'],
]);
propertyMap.forEach((lruProp, cacheProp) => {
Object.defineProperty(cache, cacheProp, {
get: function () {
return lruCache[lruProp];
},
configurable: true,
enumerable: true,
});
});
cache.set = function (key, value, ttl) {
if (!cache.enabled) {
return;
}
const opts = {};
if (ttl) {
opts.ttl = ttl;
}
cacheSet.apply(lruCache, [key, value, opts]);
};
cache.get = function (key) {
if (!cache.enabled) {
return undefined;
}
const data = lruCache.get(key);
if (data === undefined) {
cache.misses += 1;
} else {
cache.hits += 1;
}
return data;
};
cache.del = function (keys) {
if (!Array.isArray(keys)) {
keys = [keys];
}
pubsub.publish(`${cache.name}:lruCache:del`, keys);
keys.forEach(key => lruCache.delete(key));
};
cache.delete = cache.del;
cache.reset = function () {
pubsub.publish(`${cache.name}:lruCache:reset`);
localReset();
};
cache.clear = cache.reset;
function localReset() {
lruCache.clear();
cache.hits = 0;
cache.misses = 0;
}
pubsub.on(`${cache.name}:lruCache:reset`, () => {
localReset();
});
pubsub.on(`${cache.name}:lruCache:del`, (keys) => {
if (Array.isArray(keys)) {
keys.forEach(key => lruCache.delete(key));
}
});
cache.getUnCachedKeys = function (keys, cachedData) {
if (!cache.enabled) {
return keys;
}
let data;
let isCached;
const unCachedKeys = keys.filter((key) => {
data = cache.get(key);
isCached = data !== undefined;
if (isCached) {
cachedData[key] = data;
}
return !isCached;
});
const hits = keys.length - unCachedKeys.length;
const misses = keys.length - hits;
cache.hits += hits;
cache.misses += misses;
return unCachedKeys;
};
cache.dump = function () {
return lruCache.dump();
};
cache.peek = function (key) {
return lruCache.peek(key);
};
return cache;
};

94
src/cache/ttl.js vendored
Unescape Escape View File

@ -0,0 +1,94 @@
'use strict';
module.exports = function (opts) {
const TTLCache = require('@isaacs/ttlcache');
const pubsub = require('../pubsub');
const ttlCache = new TTLCache(opts);
const cache = {};
cache.name = opts.name;
cache.hits = 0;
cache.misses = 0;
cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true;
const cacheSet = ttlCache.set;
cache.set = function (key, value, ttl) {
if (!cache.enabled) {
return;
}
const opts = {};
if (ttl) {
opts.ttl = ttl;
}
cacheSet.apply(ttlCache, [key, value, opts]);
};
cache.get = function (key) {
if (!cache.enabled) {
return undefined;
}
const data = ttlCache.get(key);
if (data === undefined) {
cache.misses += 1;
} else {
cache.hits += 1;
}
return data;
};
cache.del = function (keys) {
if (!Array.isArray(keys)) {
keys = [keys];
}
pubsub.publish(`${cache.name}:ttlCache:del`, keys);
keys.forEach(key => ttlCache.delete(key));
};
cache.delete = cache.del;
cache.reset = function () {
pubsub.publish(`${cache.name}:ttlCache:reset`);
localReset();
};
cache.clear = cache.reset;
function localReset() {
ttlCache.clear();
cache.hits = 0;
cache.misses = 0;
}
pubsub.on(`${cache.name}:ttlCache:reset`, () => {
localReset();
});
pubsub.on(`${cache.name}:ttlCache:del`, (keys) => {
if (Array.isArray(keys)) {
keys.forEach(key => ttlCache.delete(key));
}
});
cache.getUnCachedKeys = function (keys, cachedData) {
if (!cache.enabled) {
return keys;
}
let data;
let isCached;
const unCachedKeys = keys.filter((key) => {
data = cache.get(key);
isCached = data !== undefined;
if (isCached) {
cachedData[key] = data;
}
return !isCached;
});
const hits = keys.length - unCachedKeys.length;
const misses = keys.length - hits;
cache.hits += hits;
cache.misses += misses;
return unCachedKeys;
};
return cache;
};

142
src/cacheCreate.js
Unescape Escape View File

@ -1,143 +1,3 @@
'use strict';
module.exports = function (opts) {
const LRU = require('lru-cache');
const pubsub = require('./pubsub');
// lru-cache@7 deprecations
const winston = require('winston');
const chalk = require('chalk');
// sometimes we kept passing in `length` with no corresponding `maxSize`.
// This is now enforced in v7; drop superfluous property
if (opts.hasOwnProperty('length') && !opts.hasOwnProperty('maxSize')) {
winston.warn(`[cache/init(${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} ${chalk.yellow('length')} was passed in without a corresponding ${chalk.yellow('maxSize')}. Both are now required as of lru-cache@7.0.0.`);
delete opts.length;
}
const deprecations = new Map([
['stale', 'allowStale'],
['maxAge', 'ttl'],
['length', 'sizeCalculation'],
]);
deprecations.forEach((newProp, oldProp) => {
if (opts.hasOwnProperty(oldProp) && !opts.hasOwnProperty(newProp)) {
winston.warn(`[cache/init (${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} The option ${chalk.yellow(oldProp)} has been deprecated as of lru-cache@7.0.0. Please change this to ${chalk.yellow(newProp)} instead.`);
opts[newProp] = opts[oldProp];
delete opts[oldProp];
}
});
const lruCache = new LRU(opts);
const cache = {};
cache.name = opts.name;
cache.hits = 0;
cache.misses = 0;
cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true;
const cacheSet = lruCache.set;
// backwards compatibility
const propertyMap = new Map([
['length', 'calculatedSize'],
['max', 'max'],
['maxSize', 'maxSize'],
['itemCount', 'size'],
]);
propertyMap.forEach((lruProp, cacheProp) => {
Object.defineProperty(cache, cacheProp, {
get: function () {
return lruCache[lruProp];
},
configurable: true,
enumerable: true,
});
});
cache.set = function (key, value, ttl) {
if (!cache.enabled) {
return;
}
const opts = {};
if (ttl) {
opts.ttl = ttl;
}
cacheSet.apply(lruCache, [key, value, opts]);
};
cache.get = function (key) {
if (!cache.enabled) {
return undefined;
}
const data = lruCache.get(key);
if (data === undefined) {
cache.misses += 1;
} else {
cache.hits += 1;
}
return data;
};
cache.del = function (keys) {
if (!Array.isArray(keys)) {
keys = [keys];
}
pubsub.publish(`${cache.name}:cache:del`, keys);
keys.forEach(key => lruCache.delete(key));
};
cache.delete = cache.del;
cache.reset = function () {
pubsub.publish(`${cache.name}:cache:reset`);
localReset();
};
cache.clear = cache.reset;
function localReset() {
lruCache.clear();
cache.hits = 0;
cache.misses = 0;
}
pubsub.on(`${cache.name}:cache:reset`, () => {
localReset();
});
pubsub.on(`${cache.name}:cache:del`, (keys) => {
if (Array.isArray(keys)) {
keys.forEach(key => lruCache.delete(key));
}
});
cache.getUnCachedKeys = function (keys, cachedData) {
if (!cache.enabled) {
return keys;
}
let data;
let isCached;
const unCachedKeys = keys.filter((key) => {
data = cache.get(key);
isCached = data !== undefined;
if (isCached) {
cachedData[key] = data;
}
return !isCached;
});
const hits = keys.length - unCachedKeys.length;
const misses = keys.length - hits;
cache.hits += hits;
cache.misses += misses;
return unCachedKeys;
};
cache.dump = function () {
return lruCache.dump();
};
cache.peek = function (key) {
return lruCache.peek(key);
};
return cache;
};
module.exports = require('./cache/lru');

2
src/database/cache.js
Unescape Escape View File

@ -1,7 +1,7 @@
'use strict';
module.exports.create = function (name) {
const cacheCreate = require('../cacheCreate');
const cacheCreate = require('../cache/lru');
return cacheCreate({
name: `${name}-object`,
max: 40000,

2
src/groups/cache.js
Unescape Escape View File

@ -1,6 +1,6 @@
'use strict';
const cacheCreate = require('../cacheCreate');
const cacheCreate = require('../cache/lru');
module.exports = function (Groups) {
Groups.cache = cacheCreate({

2
src/middleware/index.js
Unescape Escape View File

@ -14,7 +14,7 @@ const user = require('../user');
const groups = require('../groups');
const analytics = require('../analytics');
const privileges = require('../privileges');
const cacheCreate = require('../cacheCreate');
const cacheCreate = require('../cache/lru');
const helpers = require('./helpers');
const controllers = {

4
src/middleware/uploads.js
Unescape Escape View File

@ -1,6 +1,6 @@
'use strict';
const cacheCreate = require('../cacheCreate');
const cacheCreate = require('../cache/ttl');
const meta = require('../meta');
const helpers = require('./helpers');
const user = require('../user');
@ -19,7 +19,7 @@ exports.ratelimit = helpers.try(async (req, res, next) => {
return next();
}
const count = (cache.peek(`${req.ip}:uploaded_file_count`) || 0) + req.files.files.length;
const count = (cache.get(`${req.ip}:uploaded_file_count`) || 0) + req.files.files.length;
if (count > meta.config.uploadRateLimitThreshold) {
return next(new Error(['[[error:upload-ratelimit-reached]]']));
}

2
src/posts/cache.js
Unescape Escape View File

@ -1,6 +1,6 @@
'use strict';
const cacheCreate = require('../cacheCreate');
const cacheCreate = require('../cache/lru');
const meta = require('../meta');
module.exports = cacheCreate({

2
src/user/blocks.js
Unescape Escape View File

@ -2,7 +2,7 @@
const db = require('../database');
const plugins = require('../plugins');
const cacheCreate = require('../cacheCreate');
const cacheCreate = require('../cache/lru');
module.exports = function (User) {
User.blocks = {

Write Preview
Loading…
Cancel
Save