From f6be3eacfc4d972b5ca16cfdbc36972ffdddbf8c Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 23 Oct 2013 12:37:29 -0400 Subject: [PATCH 1/5] interim commit --- public/templates/header.tpl | 3 --- src/notifications.js | 9 +++++++++ src/upgrade.js | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/public/templates/header.tpl b/public/templates/header.tpl index 100d2d71df..f5e8f60c22 100644 --- a/public/templates/header.tpl +++ b/public/templates/header.tpl @@ -24,9 +24,6 @@ "forum": '../forum' } }); - requirejs.onError = function(err) { - console.log(err); - } diff --git a/src/notifications.js b/src/notifications.js index d6b2791028..e327e4c59f 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -30,6 +30,7 @@ var RDB = require('./redis.js'), * the new one put in its place. */ RDB.incr('notifications:next_nid', function(err, nid) { + RDB.sadd('notifications', nid); RDB.hmset('notifications:' + nid, { text: text || '', path: path || null, @@ -124,6 +125,14 @@ var RDB = require('./redis.js'), }); } else callback(); }); + }, + prune: function(cutoff, callback) { + var today = new Date(); + if (!cutoff || !(cutoff instanceof Date)) cutoff = new Date(today.getFullYear(), today.getMonth(), today.getDate() - 7); + + RDB.smembers('notifications', function(err, nids) { + + }); } } diff --git a/src/upgrade.js b/src/upgrade.js index 9a6faed535..3c86eea4f6 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -59,6 +59,27 @@ Upgrade.upgrade = function() { next(); } }); + }, + function(next) { + RDB.exists('notifications', function(err, exists) { + if (!exists) { + RDB.keys('notifications:*', function(err, keys) { + var multi = RDB.multi(); + + keys = keys.filter(function(key) { + if (key === 'notifications:next_nid') return false; + else return true; + }).map(function(key) { + return key.slice(14); + }); + + winston.info('[2013/10/23] Adding existing notifications to set'); + RDB.sadd('notifications', keys, next); + }); + } else { + winston.info('[2013/10/23] Updates to Notifications skipped.'); + } + }); } // Add new schema updates here ], function(err) { From aecbe6d316db6da74cbd09629e96984c56ecde33 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 25 Oct 2013 16:01:31 -0400 Subject: [PATCH 2/5] interim commit, still nothing done --- src/notifications.js | 16 +++++++++++++--- src/routes/debug.js | 14 ++++++++++++++ src/webserver.js | 4 ++++ 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 src/routes/debug.js diff --git a/src/notifications.js b/src/notifications.js index e327e4c59f..fe92a1c9ed 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -128,10 +128,19 @@ var RDB = require('./redis.js'), }, prune: function(cutoff, callback) { var today = new Date(); - if (!cutoff || !(cutoff instanceof Date)) cutoff = new Date(today.getFullYear(), today.getMonth(), today.getDate() - 7); + if (!cutoff) cutoff = new Date(today.getFullYear(), today.getMonth(), today.getDate() - 7); - RDB.smembers('notifications', function(err, nids) { + var cutoffTime = cutoff.getTime(); + RDB.smembers('notifications', function(err, nids) { + async.filter(nids, function(nid, next) { + RDB.hget('notifications:' + nid, 'datetime', function(err, datetime) { + if (parseInt(datetime, 10) < cutoffTime) next(true); + else next(false); + }); + }, function(expiredNids) { + console.log(expiredNids); + }); }); } } @@ -141,5 +150,6 @@ module.exports = { create: notifications.create, push: notifications.push, mark_read: notifications.mark_read_multiple, - mark_all_read: notifications.mark_all_read + mark_all_read: notifications.mark_all_read, + prune: notifications.prune } \ No newline at end of file diff --git a/src/routes/debug.js b/src/routes/debug.js new file mode 100644 index 0000000000..5b9298c42e --- /dev/null +++ b/src/routes/debug.js @@ -0,0 +1,14 @@ +var DebugRoute = function(app) { + var Notifications = require('../notifications'); + + app.namespace('/debug', function() { + app.get('/prune', function(req, res) { + Notifications.prune(new Date(), function() { + console.log('done'); + }); + res.send(); + }); + }); + }; + +module.exports = DebugRoute; \ No newline at end of file diff --git a/src/webserver.js b/src/webserver.js index e216c6ebfa..b660f34372 100644 --- a/src/webserver.js +++ b/src/webserver.js @@ -656,6 +656,10 @@ var express = require('express'), }); }); + // Debug routes + if (process.env.NODE_ENV === 'development') { + require('./routes/debug')(app); + } var custom_routes = { 'routes': [], From ed7c9348b77d9c5731ca31445f5b09625276359a Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 28 Oct 2013 13:45:32 -0400 Subject: [PATCH 3/5] interim commit --- src/notifications.js | 50 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/src/notifications.js b/src/notifications.js index fe92a1c9ed..4ff52af1a8 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -132,16 +132,48 @@ var RDB = require('./redis.js'), var cutoffTime = cutoff.getTime(); - RDB.smembers('notifications', function(err, nids) { - async.filter(nids, function(nid, next) { - RDB.hget('notifications:' + nid, 'datetime', function(err, datetime) { - if (parseInt(datetime, 10) < cutoffTime) next(true); - else next(false); + async.parallel({ + "inboxes": function(next) { + RDB.keys('uid:*:notifications:unread', next); + }, + "nids": function(next) { + RDB.smembers('notifications', function(err, nids) { + async.filter(nids, function(nid, next) { + RDB.hget('notifications:' + nid, 'datetime', function(err, datetime) { + if (parseInt(datetime, 10) < cutoffTime) next(true); + else next(false); + }); + }, function(expiredNids) { + next(null, expiredNids); + }); }); - }, function(expiredNids) { - console.log(expiredNids); - }); - }); + } + }, function(err, results) { + if (!err) { + var numInboxes = results.inboxes.length, + x; + + async.each(results.nids, function(nid, next) { + var multi = RDB.multi(); + + for(x=0;x Date: Mon, 28 Oct 2013 14:53:41 -0400 Subject: [PATCH 4/5] completed notifications pruning method --- public/src/forum/footer.js | 6 ++++-- src/notifications.js | 31 ++++++++++++++++++++++++++++--- src/user.js | 16 ++++++++++++++-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/public/src/forum/footer.js b/public/src/forum/footer.js index 4c05432027..aa94b302c5 100644 --- a/public/src/forum/footer.js +++ b/public/src/forum/footer.js @@ -109,6 +109,7 @@ numUnread = data.unread.length, x; notifList.innerHTML = ''; + console.log(data); if ((data.read.length + data.unread.length) > 0) { for (x = 0; x < numUnread; x++) { notifEl.setAttribute('data-nid', data.unread[x].nid); @@ -123,15 +124,16 @@ notifFrag.appendChild(notifEl.cloneNode(true)); } } else { + notifEl.className = 'no-notifs'; notifEl.innerHTML = 'You have no notifications'; - notifFrag.appendChild(notifEl); + notifFrag.appendChild(notifEl.cloneNode(true)); } // Add dedicated link to /notifications notifEl.removeAttribute('data-nid'); notifEl.className = 'pagelink'; notifEl.innerHTML = 'See all Notifications'; - notifFrag.appendChild(notifEl); + notifFrag.appendChild(notifEl.cloneNode(true)); notifList.appendChild(notifFrag); diff --git a/src/notifications.js b/src/notifications.js index 4ff52af1a8..2cf1aa37ab 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -7,10 +7,13 @@ var RDB = require('./redis.js'), RDB.multi() .hmget('notifications:' + nid, 'text', 'score', 'path', 'datetime', 'uniqueId') .zrank('uid:' + uid + ':notifications:read', nid) + .exists('notifications:' + nid) .exec(function(err, results) { var notification = results[0] readIdx = results[1]; + if (!results[2]) return callback(null); + callback({ nid: nid, text: notification[0], @@ -41,6 +44,19 @@ var RDB = require('./redis.js'), }); }); }, + destroy: function(nid) { + var multi = RDB.multi(); + + multi.del('notifications:' + nid); + multi.srem('notifications', nid); + + multi.exec(function(err) { + if (err) { + winston.error('Problem deleting expired notifications. Stack follows.'); + winston.error(err.stack); + } + }); + }, push: function(nid, uids, callback) { if (!Array.isArray(uids)) uids = [uids]; @@ -153,15 +169,24 @@ var RDB = require('./redis.js'), var numInboxes = results.inboxes.length, x; - async.each(results.nids, function(nid, next) { + async.eachSeries(results.nids, function(nid, next) { var multi = RDB.multi(); for(x=0;x 0) { async.eachSeries(nids, function(nid, next) { notifications.get(nid, uid, function(notif_data) { - unread.push(notif_data); + // If the notification could not be found, silently drop it + if (notif_data) { + unread.push(notif_data); + } else { + RDB.zrem('uid:' + uid + ':notifications:unread', nid); + } + next(); }); }, function(err) { @@ -925,7 +931,13 @@ var utils = require('./../public/src/utils.js'), if (nids && nids.length > 0) { async.eachSeries(nids, function(nid, next) { notifications.get(nid, uid, function(notif_data) { - read.push(notif_data); + // If the notification could not be found, silently drop it + if (notif_data) { + read.push(notif_data); + } else { + RDB.zrem('uid:' + uid + ':notifications:read', nid); + } + next(); }); }, function(err) { From 26d9cc56d30c6a0a3c75cd427f04c71a67fb592a Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 28 Oct 2013 15:10:10 -0400 Subject: [PATCH 5/5] added cronjob for notifications --- app.js | 5 +- package.json | 3 +- src/notifications.js | 108 ++++++++++++++++++++++++++++++++----------- 3 files changed, 88 insertions(+), 28 deletions(-) diff --git a/app.js b/app.js index 05ba01c22f..075a029fbe 100644 --- a/app.js +++ b/app.js @@ -87,7 +87,8 @@ SocketIO = require('socket.io').listen(global.server, { log: false, transports: ['websocket', 'xhr-polling', 'jsonp-polling', 'flashsocket']}), websockets = require('./src/websockets.js'), posts = require('./src/posts.js'), - plugins = require('./src/plugins'); // Don't remove this - plugins initializes itself + plugins = require('./src/plugins'), // Don't remove this - plugins initializes itself + Notifications = require('./src/notifications'); websockets.init(SocketIO); @@ -106,6 +107,8 @@ ]); templates.ready(webserver.init); + + Notifications.init(); }); } else if (nconf.get('upgrade')) { diff --git a/package.json b/package.json index 89d3e9b69c..4890e1acd5 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "nodebb-plugin-mentions": "~0.1.13", "nodebb-plugin-markdown": "~0.1.7", "nodebb-theme-vanilla": "designcreateplay/nodebb-theme-vanilla", - "nodebb-theme-cerulean": "0.0.3" + "nodebb-theme-cerulean": "0.0.3", + "cron": "~1.0.1" }, "optionalDependencies": { "hiredis": "~0.1.15" diff --git a/src/notifications.js b/src/notifications.js index 2cf1aa37ab..f6d4f03e06 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -1,18 +1,29 @@ var RDB = require('./redis.js'), async = require('async'), utils = require('../public/src/utils.js'), + winston = require('winston'), + cron = require('cron').CronJob, notifications = { + init: function() { + if (process.env.NODE_ENV === 'development') { + winston.info('[notifications.init] Registering jobs.'); + } + + new cron('0 0 * * *', notifications.prune, null, true); + }, get: function(nid, uid, callback) { RDB.multi() .hmget('notifications:' + nid, 'text', 'score', 'path', 'datetime', 'uniqueId') .zrank('uid:' + uid + ':notifications:read', nid) .exists('notifications:' + nid) .exec(function(err, results) { - var notification = results[0] + var notification = results[0], readIdx = results[1]; - if (!results[2]) return callback(null); + if (!results[2]) { + return callback(null); + } callback({ nid: nid, @@ -40,7 +51,9 @@ var RDB = require('./redis.js'), datetime: Date.now(), uniqueId: uniqueId || utils.generateUUID() }, function(err, status) { - if (!err) callback(nid); + if (!err) { + callback(nid); + } }); }); }, @@ -65,12 +78,14 @@ var RDB = require('./redis.js'), notifications.get(nid, null, function(notif_data) { for (x = 0; x < numUids; x++) { - if (parseInt(uids[x]) > 0) { + if (parseInt(uids[x], 10) > 0) { (function(uid) { notifications.remove_by_uniqueId(notif_data.uniqueId, uid, function() { RDB.zadd('uid:' + uid + ':notifications:unread', notif_data.datetime, nid); global.io.sockets.in('uid_' + uid).emit('event:new_notification'); - if (callback) callback(true); + if (callback) { + callback(true); + } }); })(uids[x]); } @@ -84,13 +99,18 @@ var RDB = require('./redis.js'), if (nids && nids.length > 0) { async.each(nids, function(nid, next) { notifications.get(nid, uid, function(nid_info) { - if (nid_info.uniqueId === uniqueId) RDB.zrem('uid:' + uid + ':notifications:unread', nid); + if (nid_info.uniqueId === uniqueId) { + RDB.zrem('uid:' + uid + ':notifications:unread', nid); + } + next(); }); }, function(err) { next(); }); - } else next(); + } else { + next(); + } }); }, function(next) { @@ -98,17 +118,24 @@ var RDB = require('./redis.js'), if (nids && nids.length > 0) { async.each(nids, function(nid, next) { notifications.get(nid, uid, function(nid_info) { - if (nid_info.uniqueId === uniqueId) RDB.zrem('uid:' + uid + ':notifications:read', nid); + if (nid_info.uniqueId === uniqueId) { + RDB.zrem('uid:' + uid + ':notifications:read', nid); + } + next(); }); }, function(err) { next(); }); - } else next(); + } else { + next(); + } }); } ], function(err) { - if (!err) callback(true); + if (!err) { + callback(true); + } }); }, mark_read: function(nid, uid, callback) { @@ -116,35 +143,55 @@ var RDB = require('./redis.js'), notifications.get(nid, uid, function(notif_data) { RDB.zrem('uid:' + uid + ':notifications:unread', nid); RDB.zadd('uid:' + uid + ':notifications:read', notif_data.datetime, nid); - if (callback) callback(); + if (callback) { + callback(); + } }); } }, mark_read_multiple: function(nids, uid, callback) { - if (!Array.isArray(nids) && parseInt(nids, 10) > 0) nids = [nids]; + if (!Array.isArray(nids) && parseInt(nids, 10) > 0) { + nids = [nids]; + } async.each(nids, function(nid, next) { notifications.mark_read(nid, uid, function(err) { - if (!err) next(null); + if (!err) { + next(null); + } }); }, function(err) { - if (callback) callback(err); + if (callback) { + callback(err); + } }); }, mark_all_read: function(uid, callback) { RDB.zrange('uid:' + uid + ':notifications:unread', 0, 10, function(err, nids) { - if (err) return callback(err); + if (err) { + return callback(err); + } if (nids.length > 0) { notifications.mark_read_multiple(nids, uid, function(err) { callback(err); }); - } else callback(); + } else { + callback(); + } }); }, - prune: function(cutoff, callback) { - var today = new Date(); - if (!cutoff) cutoff = new Date(today.getFullYear(), today.getMonth(), today.getDate() - 7); + prune: function(cutoff) { + if (process.env.NODE_ENV === 'development') { + winston.info('[notifications.prune] Removing expired notifications from the database.'); + } + + var today = new Date(), + numPruned = 0; + + if (!cutoff) { + cutoff = new Date(today.getFullYear(), today.getMonth(), today.getDate() - 7); + } var cutoffTime = cutoff.getTime(); @@ -156,8 +203,11 @@ var RDB = require('./redis.js'), RDB.smembers('notifications', function(err, nids) { async.filter(nids, function(nid, next) { RDB.hget('notifications:' + nid, 'datetime', function(err, datetime) { - if (parseInt(datetime, 10) < cutoffTime) next(true); - else next(false); + if (parseInt(datetime, 10) < cutoffTime) { + next(true); + } else { + next(false); + } }); }, function(expiredNids) { next(null, expiredNids); @@ -179,17 +229,22 @@ var RDB = require('./redis.js'), multi.exec(function(err, results) { // If the notification is not present in any inbox, delete it altogether var expired = results.every(function(present) { - if (present === null) return true; + if (present === null) { + return true; + } }); if (expired) { notifications.destroy(nid); + numPruned++; } next(); }); }, function(err) { - + if (process.env.NODE_ENV === 'development') { + winston.info('[notifications.prune] Notification pruning completed. ' + numPruned + ' expired notification' + (numPruned !== 1 ? 's' : '') + ' removed.'); + } }); } else { if (process.env.NODE_ENV === 'development') { @@ -197,16 +252,17 @@ var RDB = require('./redis.js'), winston.error(err.stack); } } - }) + }); } - } + }; module.exports = { + init: notifications.init, get: notifications.get, create: notifications.create, push: notifications.push, mark_read: notifications.mark_read_multiple, mark_all_read: notifications.mark_all_read, prune: notifications.prune -} \ No newline at end of file +};