upgrade tags to sorted set

src/controllers/tags.js

@@ -27,29 +27,13 @@ tagsController.getTag = function(req, res, next) {
 };
 
 tagsController.getTags = function(req, res, next) {
-	topics.getTagsObjects(function(err, tags) {
+	topics.getTags(0, -1, function(err, tags) {
 		if (err) {
 			return next(err);
 		}
 
-		async.map(tags, function(tag, next) {
-			topics.getTagTopicCount(tag.name, function(err, count) {
-				if (err) {
-					return next(err);
-				}
-				tag.topicCount = count;
-				next(null, tag);
-			});
-		}, function(err, tags) {
-			if (err) {
-				return next(err);
-			}
-			tags = tags.sort(function(a, b) {
-				return parseInt(b.topicCount, 10) - parseInt(a.topicCount, 10);
-			});
 		res.render('tags', {tags: tags});
 	});
-	});
 
 };
 

src/database/level/sorted.js

@@ -57,6 +57,10 @@ module.exports = function(db, module) {
 		});
 	};
 
+	module.getSortedSetRevRangeWithScores = function(key, start, stop, callback) {
+		// should return [{value:"test", score: 2}, {value: "foo", score: 1}, ...]
+	};
+
 	module.getSortedSetRangeByScore = function(key, start, count, min, max, callback) {
 		module.getListRange(key, 0, -1, function(err, list) {
 			if (min && max) {

src/database/mongo/sorted.js

@@ -19,8 +19,8 @@ module.exports = function(db, module) {
 		db.collection('objects').remove({_key:key, value:value}, helpers.done(callback));
 	};
 
-	function getSortedSetRange(key, start, stop, sort, callback) {
+	function getSortedSetRange(key, start, stop, sort, withScores, callback) {
-		db.collection('objects').find({_key:key}, {fields:{value:1}})
+		db.collection('objects').find({_key:key}, {fields: {_id: 0, value: 1, score: 1}})
 			.limit(stop - start + 1)
 			.skip(start)
 			.sort({score: sort})
@@ -29,20 +29,26 @@ module.exports = function(db, module) {
 					return callback(err, null);
 				}
 
+				if (!withScores) {
 					data = data.map(function(item) {
 						return item.value;
 					});
+				}
 
 				callback(null, data);
 			});
 	}
 
 	module.getSortedSetRange = function(key, start, stop, callback) {
-		getSortedSetRange(key, start, stop, 1, callback);
+		getSortedSetRange(key, start, stop, 1, false, callback);
 	};
 
 	module.getSortedSetRevRange = function(key, start, stop, callback) {
-		getSortedSetRange(key, start, stop, -1, callback);
+		getSortedSetRange(key, start, stop, -1, false, callback);
+	};
+
+	module.getSortedSetRevRangeWithScores = function(key, start, stop, callback) {
+		getSortedSetRange(key, start, stop, -1, true, callback)
 	};
 
 	module.getSortedSetRangeByScore = function(key, start, count, min, max, callback) {

src/database/redis/sorted.js

@@ -17,6 +17,19 @@ module.exports = function(redisClient, module) {
 		redisClient.zrevrange(key, start, stop, callback);
 	};
 
+	module.getSortedSetRevRangeWithScores = function(key, start, stop, callback) {
+		redisClient.zrevrange([key, start, stop, 'WITHSCORES'], function(err, data) {
+			if (err) {
+				return callback(err);
+			}
+			var objects = [];
+			for(var i=0; i<data.length; i+=2) {
+				objects.push({value: data[i], score: data[i+1]});
+			}
+			callback(null, objects);
+		});
+	};
+
 	module.getSortedSetRangeByScore = function(key, start, count, min, max, callback) {
 		redisClient.zrangebyscore([key, min, max, 'LIMIT', start, count], callback);
 	};

src/topics/create.js

@@ -54,14 +54,15 @@ module.exports = function(Topics) {
 				db.sortedSetAdd('topics:tid', timestamp, tid);
 				plugins.fireHook('action:topic.save', tid);
 
-				Topics.createTags(data.tags, tid, timestamp);
 				user.addTopicIdToUser(uid, tid, timestamp);
 
 				db.sortedSetAdd('categories:' + cid + ':tid', timestamp, tid);
 				db.incrObjectField('category:' + cid, 'topic_count');
 				db.incrObjectField('global', 'topicCount');
 
-				callback(null, tid);
+				Topics.createTags(data.tags, tid, timestamp, function(err) {
+					callback(err, tid);
+				});
 			});
 		});
 	};

src/topics/tags.js

@@ -7,18 +7,31 @@ var async = require('async'),
 
 module.exports = function(Topics) {
 
-	Topics.createTags = function(tags, tid, timestamp) {
+	Topics.createTags = function(tags, tid, timestamp, callback) {
 		if(Array.isArray(tags)) {
-			for (var i=0; i<tags.length; ++i) {
+			async.each(tags, function(tag, next) {
-				tags[i] = utils.removePunctuation(tags[i].trim().toLowerCase()).substr(0, 20); // TODO: make max length configurable
+				tag = utils.removePunctuation(tag.trim().toLowerCase()).substr(0, 20); // TODO: make max length configurable
 
-				db.sortedSetAdd('tag:' + tags[i] + ':topics', timestamp, tid);
+				db.setAdd('topic:' + tid + ':tags', tag);
-				db.setAdd('topic:' + tid + ':tags', tags[i]);
+
-				db.setAdd('tags', tags[i]);
+				db.sortedSetAdd('tag:' + tag + ':topics', timestamp, tid, function(err) {
+					if (!err) {
+						updateTagCount(tag);
 					}
+					next(err);
+				});
+			}, callback);
 		}
 	};
 
+	function updateTagCount(tag) {
+		Topics.getTagTopicCount(tag, function(err, count) {
+			if (!err) {
+				db.sortedSetAdd('tags:topic:count', count, tag);
+			}
+		});
+	}
+
 	Topics.getTagTids = function(tag, start, end, callback) {
 		db.getSortedSetRevRange('tag:' + tag + ':topics', start, end, callback);
 	};
@@ -29,18 +42,11 @@ module.exports = function(Topics) {
 
 	Topics.deleteTag = function(tag) {
 		db.delete('tag:' + tag + ':topics');
-		db.setRemove('tags', tag);
+		db.sortedSetRemove('tags:topic:count', tag);
-	};
-
-	Topics.getTags = function(callback) {
-		db.getSetMembers('tags', callback);
 	};
 
-	//returns tags as objects cuz templates.js cant do arrays yet >_>
+	Topics.getTags = function(start, end, callback) {
-	Topics.getTagsObjects = function(callback) {
+		db.getSortedSetRevRangeWithScores('tags:topic:count', start, end, callback);
-		Topics.getTags(function(err, tags) {
-			callback(err, mapToObject(tags));
-		});
 	};
 
 	Topics.getTopicTags = function(tid, callback) {
@@ -99,7 +105,7 @@ module.exports = function(Topics) {
 			return callback(null, []);
 		}
 
-		db.getSetMembers('tags', function(err, tags) {
+		db.getSortedSetRevRange('tags:topic:count', 0, -1, function(err, tags) {
 			if (err) {
 				return callback(null, []);
 			}

src/upgrade.js

@@ -19,7 +19,7 @@ var db = require('./database'),
 	schemaDate, thisSchemaDate,
 
 	// IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema
-	latestSchema = Date.UTC(2014, 4, 2);
+	latestSchema = Date.UTC(2014, 4, 22);
 
 Upgrade.check = function(callback) {
 	db.get('schemaDate', function(err, value) {
@@ -689,6 +689,50 @@ Upgrade.upgrade = function(callback) {
 				winston.info('[2014/5/16] Removing allowGuestPosting option - skipped');
 				next();
 			}
+		},
+		function(next) {
+			thisSchemaDate = Date.UTC(2014, 4, 22);
+
+			if (schemaDate < thisSchemaDate) {
+				db.exists('tags', function(err, exists) {
+					if (err || !exists) {
+						winston.info('[2014/5/22] Skipping tag upgrade');
+						return Upgrade.update(thisSchemaDate, next);
+					}
+
+					db.getSetMembers('tags', function(err, tags) {
+						if (err) {
+							return next(err);
+						}
+
+						async.each(tags, function(tag, next) {
+							db.sortedSetCard('tag:' + tag + ':topics', function(err, count) {
+								if (err) {
+									return next(err);
+								}
+								db.sortedSetAdd('tags:topic:count', count, tag, next);
+							});
+						}, function(err) {
+							if (err) {
+								winston.error('[2014/5/22] Error encountered while upgrading tags');
+								return next(err);
+							}
+
+							db.delete('tags', function(err) {
+								if (err) {
+									winston.error('[2014/5/22] Error encountered while upgrading tags');
+									return next(err);
+								}
+								winston.info('[2014/5/22] Tags upgraded to sorted set');
+								Upgrade.update(thisSchemaDate, next);
+							});
+						});
+					});
+				});
+			} else {
+				winston.info('[2014/5/16] Tags upgrade - skipped');
+				next();
+			}
 		}
 		// Add new schema updates here
 		// IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema IN LINE 22!!!