Merge branch 'notif_pruning'

app.js

@@ -87,7 +87,8 @@
 				SocketIO =  require('socket.io').listen(global.server, { log: false, transports: ['websocket', 'xhr-polling', 'jsonp-polling', 'flashsocket']}),
 				websockets = require('./src/websockets.js'),
 				posts = require('./src/posts.js'),
-				plugins = require('./src/plugins'); // Don't remove this - plugins initializes itself
+				plugins = require('./src/plugins'), // Don't remove this - plugins initializes itself
+				Notifications = require('./src/notifications');
 
 			websockets.init(SocketIO);
 
@@ -106,6 +107,8 @@
 			]);
 
 			templates.ready(webserver.init);
+
+			Notifications.init();
 		});
 
 	} else if (nconf.get('upgrade')) {

package.json

@@ -44,7 +44,8 @@
     "nodebb-plugin-mentions": "~0.1.13",
     "nodebb-plugin-markdown": "~0.1.7",
     "nodebb-theme-vanilla": "designcreateplay/nodebb-theme-vanilla",
-    "nodebb-theme-cerulean": "0.0.3"
+    "nodebb-theme-cerulean": "0.0.3",
+    "cron": "~1.0.1"
   },
   "optionalDependencies": {
     "hiredis": "~0.1.15"

public/src/forum/footer.js

@@ -101,6 +101,7 @@
 					numUnread = data.unread.length,
 					x;
 				notifList.innerHTML = '';
+				console.log(data);
 				if ((data.read.length + data.unread.length) > 0) {
 					for (x = 0; x < numUnread; x++) {
 						notifEl.setAttribute('data-nid', data.unread[x].nid);
@@ -115,15 +116,16 @@
 						notifFrag.appendChild(notifEl.cloneNode(true));
 					}
 				} else {
+					notifEl.className = 'no-notifs';
 					notifEl.innerHTML = '<a>You have no notifications</a>';
-					notifFrag.appendChild(notifEl);
+					notifFrag.appendChild(notifEl.cloneNode(true));
 				}
 
 				// Add dedicated link to /notifications
 				notifEl.removeAttribute('data-nid');
 				notifEl.className = 'pagelink';
 				notifEl.innerHTML = '<a href="' + RELATIVE_PATH + '/notifications">See all Notifications</a>';
-				notifFrag.appendChild(notifEl);
+				notifFrag.appendChild(notifEl.cloneNode(true));
 
 				notifList.appendChild(notifFrag);
 

public/templates/header.tpl

@@ -24,9 +24,6 @@
 				"forum": '../forum'
 			}
 		});
-		requirejs.onError = function(err) {
-			console.log(err);
-		}
 	</script>
 
 	<link rel="stylesheet" type="text/css" href="{relative_path}/css/theme.css" />

src/notifications.js

@@ -1,16 +1,30 @@
 var RDB = require('./redis.js'),
 	async = require('async'),
 	utils = require('../public/src/utils.js'),
+	winston = require('winston'),
+	cron = require('cron').CronJob,
 
 	notifications = {
+		init: function() {
+			if (process.env.NODE_ENV === 'development') {
+				winston.info('[notifications.init] Registering jobs.');
+			}
+
+			new cron('0 0 * * *', notifications.prune, null, true);
+		},
 		get: function(nid, uid, callback) {
 			RDB.multi()
 				.hmget('notifications:' + nid, 'text', 'score', 'path', 'datetime', 'uniqueId')
 				.zrank('uid:' + uid + ':notifications:read', nid)
+				.exists('notifications:' + nid)
 				.exec(function(err, results) {
-					var	notification = results[0]
+					var	notification = results[0],
 						readIdx = results[1];
 
+					if (!results[2]) {
+						return callback(null);
+					}
+
 					callback({
 						nid: nid,
 						text: notification[0],
@@ -30,16 +44,32 @@ var RDB = require('./redis.js'),
 			 *		the new one put in its place.
 			 */
 			RDB.incr('notifications:next_nid', function(err, nid) {
+				RDB.sadd('notifications', nid);
 				RDB.hmset('notifications:' + nid, {
 					text: text || '',
 					path: path || null,
 					datetime: Date.now(),
 					uniqueId: uniqueId || utils.generateUUID()
 				}, function(err, status) {
-					if (!err) callback(nid);
+					if (!err) {
+						callback(nid);
+					}
 				});
 			});
 		},
+		destroy: function(nid) {
+			var	multi = RDB.multi();
+
+			multi.del('notifications:' + nid);
+			multi.srem('notifications', nid);
+
+			multi.exec(function(err) {
+				if (err) {
+					winston.error('Problem deleting expired notifications. Stack follows.');
+					winston.error(err.stack);
+				}
+			});
+		},
 		push: function(nid, uids, callback) {
 			if (!Array.isArray(uids)) uids = [uids];
 
@@ -48,12 +78,14 @@ var RDB = require('./redis.js'),
 
 			notifications.get(nid, null, function(notif_data) {
 				for (x = 0; x < numUids; x++) {
-					if (parseInt(uids[x]) > 0) {
+					if (parseInt(uids[x], 10) > 0) {
 						(function(uid) {
 							notifications.remove_by_uniqueId(notif_data.uniqueId, uid, function() {
 								RDB.zadd('uid:' + uid + ':notifications:unread', notif_data.datetime, nid);
 								global.io.sockets.in('uid_' + uid).emit('event:new_notification');
-								if (callback) callback(true);
+								if (callback) {
+									callback(true);
+								}
 							});
 						})(uids[x]);
 					}
@@ -67,13 +99,18 @@ var RDB = require('./redis.js'),
 						if (nids && nids.length > 0) {
 							async.each(nids, function(nid, next) {
 								notifications.get(nid, uid, function(nid_info) {
-									if (nid_info.uniqueId === uniqueId) RDB.zrem('uid:' + uid + ':notifications:unread', nid);
+									if (nid_info.uniqueId === uniqueId) {
+										RDB.zrem('uid:' + uid + ':notifications:unread', nid);
+									}
+
 									next();
 								});
 							}, function(err) {
 								next();
 							});
-						} else next();
+						} else {
+							next();
+						}
 					});
 				},
 				function(next) {
@@ -81,17 +118,24 @@ var RDB = require('./redis.js'),
 						if (nids && nids.length > 0) {
 							async.each(nids, function(nid, next) {
 								notifications.get(nid, uid, function(nid_info) {
-									if (nid_info.uniqueId === uniqueId) RDB.zrem('uid:' + uid + ':notifications:read', nid);
+									if (nid_info.uniqueId === uniqueId) {
+										RDB.zrem('uid:' + uid + ':notifications:read', nid);
+									}
+
 									next();
 								});
 							}, function(err) {
 								next();
 							});
-						} else next();
+						} else {
+							next();
+						}
 					});
 				}
 			], function(err) {
-				if (!err) callback(true);
+				if (!err) {
+					callback(true);
+				}
 			});
 		},
 		mark_read: function(nid, uid, callback) {
@@ -99,38 +143,126 @@ var RDB = require('./redis.js'),
 				notifications.get(nid, uid, function(notif_data) {
 					RDB.zrem('uid:' + uid + ':notifications:unread', nid);
 					RDB.zadd('uid:' + uid + ':notifications:read', notif_data.datetime, nid);
-					if (callback) callback();
+					if (callback) {
+						callback();
+					}
 				});
 			}
 		},
 		mark_read_multiple: function(nids, uid, callback) {
-			if (!Array.isArray(nids) && parseInt(nids, 10) > 0) nids = [nids];
+			if (!Array.isArray(nids) && parseInt(nids, 10) > 0) {
+				nids = [nids];
+			}
 
 			async.each(nids, function(nid, next) {
 				notifications.mark_read(nid, uid, function(err) {
-					if (!err) next(null);
+					if (!err) {
+						next(null);
+					}
 				});
 			}, function(err) {
-				if (callback) callback(err);
+				if (callback) {
+					callback(err);
+				}
 			});
 		},
 		mark_all_read: function(uid, callback) {
 			RDB.zrange('uid:' + uid + ':notifications:unread', 0, 10, function(err, nids) {
-				if (err) return callback(err);
+				if (err) {
+					return callback(err);
+				}
 
 				if (nids.length > 0) {
 					notifications.mark_read_multiple(nids, uid, function(err) {
 						callback(err);
 					});
-				} else callback();
+				} else {
+					callback();
+				}
+			});
+		},
+		prune: function(cutoff) {
+			if (process.env.NODE_ENV === 'development') {
+				winston.info('[notifications.prune] Removing expired notifications from the database.');
+			}
+
+			var	today = new Date(),
+				numPruned = 0;
+
+			if (!cutoff) {
+				cutoff = new Date(today.getFullYear(), today.getMonth(), today.getDate() - 7);
+			}
+
+			var	cutoffTime = cutoff.getTime();
+
+			async.parallel({
+				"inboxes": function(next) {
+					RDB.keys('uid:*:notifications:unread', next);
+				},
+				"nids": function(next) {
+					RDB.smembers('notifications', function(err, nids) {
+						async.filter(nids, function(nid, next) {
+							RDB.hget('notifications:' + nid, 'datetime', function(err, datetime) {
+								if (parseInt(datetime, 10) < cutoffTime) {
+									next(true);
+								} else {
+									next(false);
+								}
+							});
+						}, function(expiredNids) {
+							next(null, expiredNids);
+						});
+					});
+				}
+			}, function(err, results) {
+				if (!err) {
+					var	numInboxes = results.inboxes.length,
+						x;
+
+					async.eachSeries(results.nids, function(nid, next) {
+						var	multi = RDB.multi();
+
+						for(x=0;x<numInboxes;x++) {
+							multi.zscore(results.inboxes[x], nid);
+						}
+
+						multi.exec(function(err, results) {
+							// If the notification is not present in any inbox, delete it altogether
+							var	expired = results.every(function(present) {
+									if (present === null) {
+										return true;
+									}
+								});
+
+							if (expired) {
+								notifications.destroy(nid);
+								numPruned++;
+							}
+
+							next();
+						});
+					}, function(err) {
+						if (process.env.NODE_ENV === 'development') {
+							winston.info('[notifications.prune] Notification pruning completed. ' + numPruned + ' expired notification' + (numPruned !== 1 ? 's' : '') + ' removed.');
+						}
+					});
+				} else {
+					if (process.env.NODE_ENV === 'development') {
+						winston.error('[notifications.prune] Ran into trouble pruning expired notifications. Stack trace to follow.');
+						winston.error(err.stack);
+					}
+				}
 			});
+
 		}
-	}
+	};
 
 module.exports = {
+	init: notifications.init,
 	get: notifications.get,
 	create: notifications.create,
 	push: notifications.push,
 	mark_read: notifications.mark_read_multiple,
-	mark_all_read: notifications.mark_all_read
+	mark_all_read: notifications.mark_all_read,
-}
+	prune: notifications.prune
+};

src/routes/debug.js

@@ -0,0 +1,14 @@
+var	DebugRoute = function(app) {
+		var	Notifications = require('../notifications');
+
+		app.namespace('/debug', function() {
+			app.get('/prune', function(req, res) {
+				Notifications.prune(new Date(), function() {
+					console.log('done');
+				});
+				res.send();
+			});
+		});
+	};
+
+module.exports = DebugRoute;

src/upgrade.js

@@ -59,6 +59,27 @@ Upgrade.upgrade = function() {
 					next();
 				}
 			});
+		},
+		function(next) {
+			RDB.exists('notifications', function(err, exists) {
+				if (!exists) {
+					RDB.keys('notifications:*', function(err, keys) {
+						var	multi = RDB.multi();
+
+						keys = keys.filter(function(key) {
+							if (key === 'notifications:next_nid') return false;
+							else return true;
+						}).map(function(key) {
+							return key.slice(14);
+						});
+
+						winston.info('[2013/10/23] Adding existing notifications to set');
+						RDB.sadd('notifications', keys, next);
+					});
+				} else {
+					winston.info('[2013/10/23] Updates to Notifications skipped.');
+				}
+			});
 		}
 		// Add new schema updates here
 	], function(err) {

src/user.js

@@ -903,7 +903,13 @@ var utils = require('./../public/src/utils.js'),
 						if (nids && nids.length > 0) {
 							async.eachSeries(nids, function(nid, next) {
 								notifications.get(nid, uid, function(notif_data) {
-									unread.push(notif_data);
+									// If the notification could not be found, silently drop it
+									if (notif_data) {
+										unread.push(notif_data);
+									} else {
+										RDB.zrem('uid:' + uid + ':notifications:unread', nid);
+									}
+
 									next();
 								});
 							}, function(err) {
@@ -925,7 +931,13 @@ var utils = require('./../public/src/utils.js'),
 						if (nids && nids.length > 0) {
 							async.eachSeries(nids, function(nid, next) {
 								notifications.get(nid, uid, function(notif_data) {
-									read.push(notif_data);
+									// If the notification could not be found, silently drop it
+									if (notif_data) {
+										read.push(notif_data);
+									} else {
+										RDB.zrem('uid:' + uid + ':notifications:read', nid);
+									}
+
 									next();
 								});
 							}, function(err) {

src/webserver.js

@@ -656,6 +656,10 @@ var express = require('express'),
 			});
 		});
 
+		// Debug routes
+		if (process.env.NODE_ENV === 'development') {
+			require('./routes/debug')(app);
+		}
 
 		var custom_routes = {
 			'routes': [],