feat: closes #8316, add more data to export profile

install/package.json

@@ -65,7 +65,7 @@
         "ipaddr.js": "^1.9.1",
         "jquery": "3.5.1",
         "jsesc": "3.0.1",
-        "json-2-csv": "^3.6.2",
+        "json2csv": "5.0.1",
         "jsonwebtoken": "^8.5.1",
         "less": "^3.11.1",
         "lodash": "^4.17.15",

public/language/en-GB/user.json

@@ -191,7 +191,7 @@
 	"consent.right_to_data_portability": "You have the Right to Data Portability",
 	"consent.right_to_data_portability_description": "You may request from us a machine-readable export of any collected data about you and your account. You can do so by clicking the appropriate button below.",
 
-	"consent.export_profile": "Export Profile (.csv)",
+	"consent.export_profile": "Export Profile (.json)",
 	"consent.export_uploads": "Export Uploaded Content (.zip)",
 	"consent.export_posts": "Export Posts (.csv)"
 }

src/controllers/admin/errors.js

@@ -1,7 +1,6 @@
 'use strict';
 
-const json2csv = require('json-2-csv').json2csv;
-const util = require('util');
+const json2csvAsync = require('json2csv').parseAsync;
 
 const meta = require('../../meta');
 const analytics = require('../../analytics');
@@ -17,12 +16,10 @@ errorsController.get = async function (req, res) {
 	res.render('admin/advanced/errors', data);
 };
 
-const json2csvAsync = util.promisify(function (data, callback) {
-	json2csv(data, (err, output) => callback(err, output));
-});
-
 errorsController.export = async function (req, res) {
 	const data = await meta.errors.get(false);
-	const csv = await json2csvAsync(data);
+	const fields = data.length ? Object.keys(data[0]) : [];
+	const opts = { fields };
+	const csv = await json2csvAsync(data, opts);
 	res.set('Content-Type', 'text/csv').set('Content-Disposition', 'attachment; filename="404.csv"').send(csv);
 };

src/controllers/user.js

@@ -1,11 +1,11 @@
 'use strict';
 
+const _ = require('lodash');
 const path = require('path');
 const fs = require('fs');
 const winston = require('winston');
-const converter = require('json-2-csv');
+const json2csvAsync = require('json2csv').parseAsync;
 const archiver = require('archiver');
-const util = require('util');
 
 const db = require('../database');
 const user = require('../user');
@@ -85,10 +85,6 @@ userController.getUserDataByUID = async function (callerUid, uid) {
 	return userData;
 };
 
-const json2csv = util.promisify(function (payload, options, callback) {
-	converter.json2csv(payload, callback, options);
-});
-
 userController.exportPosts = async function (req, res) {
 	var payload = [];
 	await batch.processSortedSet('uid:' + res.locals.uid + ':posts', async function (pids) {
@@ -103,10 +99,9 @@ userController.exportPosts = async function (req, res) {
 		batch: 500,
 	});
 
-	const csv = await json2csv(payload, {
-		checkSchemaDifferences: false,
-		emptyFieldValue: '',
-	});
+	const fields = payload.length ? Object.keys(payload[0]) : [];
+	const opts = { fields };
+	const csv = await json2csvAsync(payload, opts);
 	res.set('Content-Type', 'text/csv').set('Content-Disposition', 'attachment; filename="' + res.locals.uid + '_posts.csv"').send(csv);
 };
 
@@ -192,15 +187,67 @@ userController.exportUploads = function (req, res, next) {
 };
 
 userController.exportProfile = async function (req, res) {
-	const targetUid = res.locals.uid;
-	const objects = await db.getObjects(['user:' + targetUid, 'user:' + targetUid + ':settings']);
-	Object.assign(objects[0], objects[1]);
-	delete objects[0].password;
-
-	const csv = await json2csv(objects[0], {});
-	res.set('Content-Type', 'text/csv').set('Content-Disposition', 'attachment; filename="' + targetUid + '_profile.csv"').send(csv);
+	const targetUid = parseInt(res.locals.uid, 10);
+	const [userData, userSettings, ips, sessions, usernames, emails, bookmarks, watchedTopics, upvoted, downvoted, following] = await Promise.all([
+		db.getObject('user:' + targetUid),
+		db.getObject('user:' + targetUid + ':settings'),
+		user.getIPs(targetUid, 9),
+		user.auth.getSessions(targetUid, req.sessionID),
+		user.getHistory('user:' + targetUid + ':usernames'),
+		user.getHistory('user:' + targetUid + ':emails'),
+		getSetData('uid:' + targetUid + ':bookmarks', 'post:'),
+		getSetData('uid:' + targetUid + ':followed_tids', 'topic:'),
+		getSetData('uid:' + targetUid + ':upvote', 'post:'),
+		getSetData('uid:' + targetUid + ':downvote', 'post:'),
+		getSetData('following:' + targetUid, 'user:'),
+	]);
+	delete userData.password;
+	const followingData = following.map(u => ({ username: u.username, uid: u.uid }));
+
+	let chatData = [];
+	await batch.processSortedSet('uid:' + targetUid + ':chat:rooms', async (roomIds) => {
+		var result = await Promise.all(roomIds.map(roomId => getRoomMessages(targetUid, roomId)));
+		chatData = chatData.concat(_.flatten(result));
+	}, { batch: 100 });
+
+	res.set('Content-Type', 'application/json')
+		.set('Content-Disposition', 'attachment; filename="' + targetUid + '_profile.json"')
+		.send({
+			user: userData,
+			settings: userSettings,
+			ips: ips,
+			sessions: sessions,
+			usernames: usernames,
+			emails: emails,
+			messages: chatData,
+			bookmarks: bookmarks,
+			watchedTopics: watchedTopics,
+			upvoted: upvoted,
+			downvoted: downvoted,
+			following: followingData,
+		});
 };
 
+async function getRoomMessages(uid, roomId) {
+	let data = [];
+	await batch.processSortedSet('uid:' + uid + ':chat:room:' + roomId + ':mids', async (mids) => {
+		const messageData = await db.getObjects(mids.map(mid => 'message:' + mid));
+		data = data.concat(messageData.filter(m => m && m.fromuid === uid && !m.system)
+			.map(m => ({ content: m.content, timestamp: m.timestamp }))
+		);
+	}, { batch: 500 });
+	return data;
+}
+
+async function getSetData(set, keyPrefix) {
+	let data = [];
+	await batch.processSortedSet(set, async (ids) => {
+		data = data.concat(await db.getObjects(ids.map(mid => keyPrefix + mid)));
+	}, { batch: 500 });
+	return data;
+}
+
+
 require('../promisify')(userController, [
 	'getCurrentUser', 'getUserByUID', 'getUserByUsername', 'getUserByEmail',
 	'exportPosts', 'exportUploads', 'exportProfile',