fix: remove unused data from post/topic/user hashes

src/controllers/authentication.js

@@ -237,8 +237,6 @@ function continueLogin(req, res, next) {
 			return helpers.noScriptErrors(req, res, info, 403);
 		}
 
-		var passwordExpiry = userData.passwordExpiry !== undefined ? parseInt(userData.passwordExpiry, 10) : null;
-
 		// Alter user cookie depending on passed-in option
 		if (req.body.remember === 'on') {
 			var duration = 1000 * 60 * 60 * 24 * meta.config.loginDays;
@@ -249,7 +247,7 @@ function continueLogin(req, res, next) {
 			req.session.cookie.expires = false;
 		}
 
-		if (passwordExpiry && passwordExpiry < Date.now()) {
+		if (userData.passwordExpiry && userData.passwordExpiry < Date.now()) {
 			winston.verbose('[auth] Triggering password reset for uid ' + userData.uid + ' due to password policy');
 			req.session.passwordExpired = true;
 

src/posts/create.js

@@ -35,7 +35,6 @@ module.exports = function (Posts) {
 			tid: tid,
 			content: content,
 			timestamp: timestamp,
-			deleted: 0,
 		};
 
 		if (data.toPid) {

src/upgrades/1.13.0/clean_post_topic_hash.js

@@ -0,0 +1,98 @@
+'use strict';
+
+const db = require('../../database');
+const batch = require('../../batch');
+const posts = require('../../posts');
+const topics = require('../../topics');
+
+module.exports = {
+	name: 'Clean up post hash data',
+	timestamp: Date.UTC(2019, 9, 7),
+	method: async function (callback) {
+		const progress = this.progress;
+		await cleanPost(progress);
+		await cleanTopic(progress);
+		callback();
+	},
+};
+
+async function cleanPost(progress) {
+	await batch.processSortedSet('posts:pid', async function (pids) {
+		progress.incr(pids.length);
+
+		const postData = await posts.getPostsData(pids);
+		await Promise.all(postData.map(async function (post) {
+			if (!post) {
+				return;
+			}
+			const fields = [];
+			if (post.editor === '') {
+				fields.push('editor');
+			}
+			if (post.deleted === 0) {
+				fields.push('deleted');
+			}
+			if (post.edited === 0) {
+				fields.push('edited');
+			}
+
+			// cleanup legacy fields, these are not used anymore
+			const legacyFields = [
+				'show_banned', 'fav_star_class', 'relativeEditTime',
+				'post_rep', 'relativeTime', 'fav_button_class',
+				'edited-class',
+			];
+			legacyFields.forEach((field) => {
+				if (post.hasOwnProperty(field)) {
+					fields.push(field);
+				}
+			});
+
+			if (fields.length) {
+				await db.deleteObjectFields('post:' + post.pid, fields);
+			}
+		}));
+	}, {
+		batch: 500,
+		progress: progress,
+	});
+}
+
+async function cleanTopic(progress) {
+	await batch.processSortedSet('topics:tid', async function (tids) {
+		progress.incr(tids.length);
+		const topicData = await topics.getTopicsData(tids);
+		await Promise.all(topicData.map(async function (topic) {
+			if (!topic) {
+				return;
+			}
+			const fields = [];
+			if (topic.deleted === 0) {
+				fields.push('deleted');
+			}
+			if (topic.pinned === 0) {
+				fields.push('pinned');
+			}
+			if (topic.locked === 0) {
+				fields.push('locked');
+			}
+
+			// cleanup legacy fields, these are not used anymore
+			const legacyFields = [
+				'category_name', 'category_slug',
+			];
+			legacyFields.forEach((field) => {
+				if (topic.hasOwnProperty(field)) {
+					fields.push(field);
+				}
+			});
+
+			if (fields.length) {
+				await db.deleteObjectFields('topic:' + topic.tid, fields);
+			}
+		}));
+	}, {
+		batch: 500,
+		progress: progress,
+	});
+}

src/upgrades/1.13.0/cleanup_old_notifications.js

@@ -2,9 +2,10 @@
 
 const db = require('../../database');
 const batch = require('../../batch');
+const user = require('../../user');
 
 module.exports = {
-	name: 'Clean up old notifications',
+	name: 'Clean up old notifications and hash data',
 	timestamp: Date.UTC(2019, 9, 7),
 	method: async function (callback) {
 		const progress = this.progress;
@@ -16,6 +17,32 @@ module.exports = {
 				db.sortedSetsRemoveRangeByScore(uids.map(uid => 'uid:' + uid + ':notifications:unread'), '-inf', cutoffTime),
 				db.sortedSetsRemoveRangeByScore(uids.map(uid => 'uid:' + uid + ':notifications:read'), '-inf', cutoffTime),
 			]);
+			const userData = await user.getUsersData(uids);
+			await Promise.all(userData.map(async function (user) {
+				if (!user) {
+					return;
+				}
+				const fields = [];
+				['picture', 'fullname', 'location', 'birthday', 'website', 'signature', 'uploadedpicture'].forEach((field) => {
+					if (user[field] === '') {
+						fields.push(field);
+					}
+				});
+				['profileviews', 'reputation', 'postcount', 'topiccount', 'lastposttime', 'banned'].forEach((field) => {
+					if (user[field] === 0) {
+						fields.push(field);
+					}
+				});
+				if (user['icon:text']) {
+					fields.push('icon:text');
+				}
+				if (user['icon:bgColor']) {
+					fields.push('icon:bgColor');
+				}
+				if (fields.length) {
+					await db.deleteObjectFields('user:' + user.uid, fields);
+				}
+			}));
 		}, {
 			batch: 500,
 			progress: progress,

src/user/create.js

@@ -24,23 +24,20 @@ module.exports = function (User) {
 			email: data.email || '',
 			joindate: timestamp,
 			lastonline: timestamp,
-			picture: data.picture || '',
-			fullname: data.fullname || '',
-			location: data.location || '',
-			birthday: data.birthday || '',
-			website: '',
-			signature: '',
-			uploadedpicture: '',
-			profileviews: 0,
-			reputation: 0,
-			postcount: 0,
-			topiccount: 0,
-			lastposttime: 0,
-			banned: 0,
 			status: 'online',
-			gdpr_consent: data.gdpr_consent === true ? 1 : 0,
-			acceptTos: data.acceptTos === true ? 1 : 0,
 		};
+		['picture', 'fullname', 'location', 'birthday'].forEach((field) => {
+			if (data[field]) {
+				userData[field] = data[field];
+			}
+		});
+		if (data.gdpr_consent === true) {
+			userData.gdpr_consent = 1;
+		}
+		if (data.acceptTos === true) {
+			userData.acceptTos = 1;
+		}
+
 		const renamedUsername = await User.uniqueUsername(userData);
 		const userNameChanged = !!renamedUsername;
 		if (userNameChanged) {

src/user/data.js

@@ -12,7 +12,7 @@ const utils = require('../utils');
 const intFields = [
 	'uid', 'postcount', 'topiccount', 'reputation', 'profileviews',
 	'banned', 'banned:expire', 'email:confirmed', 'joindate', 'lastonline', 'lastqueuetime',
-	'lastposttime', 'followingCount', 'followerCount',
+	'lastposttime', 'followingCount', 'followerCount', 'passwordExpiry',
 ];
 
 module.exports = function (User) {

src/user/reset.js

@@ -86,7 +86,9 @@ UserReset.updateExpiry = async function (uid) {
 	const oneDay = 1000 * 60 * 60 * 24;
 	const expireDays = meta.config.passwordExpiryDays;
 	const expiry = Date.now() + (oneDay * expireDays);
-	await user.setUserField(uid, 'passwordExpiry', expireDays > 0 ? expiry : 0);
+	if (expireDays > 0) {
+		await user.setUserField(uid, 'passwordExpiry', expiry);
+	}
 };
 
 UserReset.clean = async function () {

test/user.js

@@ -50,25 +50,21 @@ describe('User', function () {
 
 
 	describe('.create(), when created', function () {
-		it('should be created properly', function (done) {
+		it('should be created properly', async function () {
-			User.create({ username: userData.username, password: userData.password, email: userData.email }, function (error, userId) {
+			testUid = await User.create({ username: userData.username, password: userData.password, email: userData.email });
-				assert.equal(error, null, 'was created with error');
+			assert.ok(testUid);
-				assert.ok(userId);
+		});
-
+
-				testUid = userId;
+		it('should be created properly', async function () {
-				done();
+			const uid = await User.create({ username: 'weirdemail', email: '<h1>test</h1>@gmail.com' });
-			});
+			const data = await User.getUserData(uid);
-		});
+			assert.equal(data.email, '&lt;h1&gt;test&lt;&#x2F;h1&gt;@gmail.com');
-
+			assert.strictEqual(data.profileviews, 0);
-		it('should be created properly', function (done) {
+			assert.strictEqual(data.reputation, 0);
-			User.create({ username: 'weirdemail', email: '<h1>test</h1>@gmail.com' }, function (err, uid) {
+			assert.strictEqual(data.postcount, 0);
-				assert.ifError(err);
+			assert.strictEqual(data.topiccount, 0);
-				User.getUserData(uid, function (err, data) {
+			assert.strictEqual(data.lastposttime, 0);
-					assert.ifError(err);
+			assert.strictEqual(data.banned, 0);
-					assert.equal(data.email, '&lt;h1&gt;test&lt;&#x2F;h1&gt;@gmail.com');
-					done();
-				});
-			});
 		});
 
 		it('should have a valid email, if using an email', function (done) {