From f0323b6cfa19161e7b8f72543a3adade1c84fb84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Fri, 15 May 2020 15:29:28 -0400 Subject: [PATCH] feat: closes #8316, add more data to export profile --- install/package.json | 2 +- public/language/en-GB/user.json | 2 +- src/controllers/admin/errors.js | 11 ++--- src/controllers/user.js | 81 ++++++++++++++++++++++++++------- 4 files changed, 70 insertions(+), 26 deletions(-) diff --git a/install/package.json b/install/package.json index 8063b62b64..4aebe67aac 100644 --- a/install/package.json +++ b/install/package.json @@ -65,7 +65,7 @@ "ipaddr.js": "^1.9.1", "jquery": "3.5.1", "jsesc": "3.0.1", - "json-2-csv": "^3.6.2", + "json2csv": "5.0.1", "jsonwebtoken": "^8.5.1", "less": "^3.11.1", "lodash": "^4.17.15", diff --git a/public/language/en-GB/user.json b/public/language/en-GB/user.json index 18d17aeb18..f1c3b8f7fe 100644 --- a/public/language/en-GB/user.json +++ b/public/language/en-GB/user.json @@ -191,7 +191,7 @@ "consent.right_to_data_portability": "You have the Right to Data Portability", "consent.right_to_data_portability_description": "You may request from us a machine-readable export of any collected data about you and your account. You can do so by clicking the appropriate button below.", - "consent.export_profile": "Export Profile (.csv)", + "consent.export_profile": "Export Profile (.json)", "consent.export_uploads": "Export Uploaded Content (.zip)", "consent.export_posts": "Export Posts (.csv)" } diff --git a/src/controllers/admin/errors.js b/src/controllers/admin/errors.js index accefbdae2..f609ecca3d 100644 --- a/src/controllers/admin/errors.js +++ b/src/controllers/admin/errors.js @@ -1,7 +1,6 @@ 'use strict'; -const json2csv = require('json-2-csv').json2csv; -const util = require('util'); +const json2csvAsync = require('json2csv').parseAsync; const meta = require('../../meta'); const analytics = require('../../analytics'); @@ -17,12 +16,10 @@ errorsController.get = async function (req, res) { res.render('admin/advanced/errors', data); }; -const json2csvAsync = util.promisify(function (data, callback) { - json2csv(data, (err, output) => callback(err, output)); -}); - errorsController.export = async function (req, res) { const data = await meta.errors.get(false); - const csv = await json2csvAsync(data); + const fields = data.length ? Object.keys(data[0]) : []; + const opts = { fields }; + const csv = await json2csvAsync(data, opts); res.set('Content-Type', 'text/csv').set('Content-Disposition', 'attachment; filename="404.csv"').send(csv); }; diff --git a/src/controllers/user.js b/src/controllers/user.js index 5bd643d63e..8c172808fd 100644 --- a/src/controllers/user.js +++ b/src/controllers/user.js @@ -1,11 +1,11 @@ 'use strict'; +const _ = require('lodash'); const path = require('path'); const fs = require('fs'); const winston = require('winston'); -const converter = require('json-2-csv'); +const json2csvAsync = require('json2csv').parseAsync; const archiver = require('archiver'); -const util = require('util'); const db = require('../database'); const user = require('../user'); @@ -85,10 +85,6 @@ userController.getUserDataByUID = async function (callerUid, uid) { return userData; }; -const json2csv = util.promisify(function (payload, options, callback) { - converter.json2csv(payload, callback, options); -}); - userController.exportPosts = async function (req, res) { var payload = []; await batch.processSortedSet('uid:' + res.locals.uid + ':posts', async function (pids) { @@ -103,10 +99,9 @@ userController.exportPosts = async function (req, res) { batch: 500, }); - const csv = await json2csv(payload, { - checkSchemaDifferences: false, - emptyFieldValue: '', - }); + const fields = payload.length ? Object.keys(payload[0]) : []; + const opts = { fields }; + const csv = await json2csvAsync(payload, opts); res.set('Content-Type', 'text/csv').set('Content-Disposition', 'attachment; filename="' + res.locals.uid + '_posts.csv"').send(csv); }; @@ -192,15 +187,67 @@ userController.exportUploads = function (req, res, next) { }; userController.exportProfile = async function (req, res) { - const targetUid = res.locals.uid; - const objects = await db.getObjects(['user:' + targetUid, 'user:' + targetUid + ':settings']); - Object.assign(objects[0], objects[1]); - delete objects[0].password; - - const csv = await json2csv(objects[0], {}); - res.set('Content-Type', 'text/csv').set('Content-Disposition', 'attachment; filename="' + targetUid + '_profile.csv"').send(csv); + const targetUid = parseInt(res.locals.uid, 10); + const [userData, userSettings, ips, sessions, usernames, emails, bookmarks, watchedTopics, upvoted, downvoted, following] = await Promise.all([ + db.getObject('user:' + targetUid), + db.getObject('user:' + targetUid + ':settings'), + user.getIPs(targetUid, 9), + user.auth.getSessions(targetUid, req.sessionID), + user.getHistory('user:' + targetUid + ':usernames'), + user.getHistory('user:' + targetUid + ':emails'), + getSetData('uid:' + targetUid + ':bookmarks', 'post:'), + getSetData('uid:' + targetUid + ':followed_tids', 'topic:'), + getSetData('uid:' + targetUid + ':upvote', 'post:'), + getSetData('uid:' + targetUid + ':downvote', 'post:'), + getSetData('following:' + targetUid, 'user:'), + ]); + delete userData.password; + const followingData = following.map(u => ({ username: u.username, uid: u.uid })); + + let chatData = []; + await batch.processSortedSet('uid:' + targetUid + ':chat:rooms', async (roomIds) => { + var result = await Promise.all(roomIds.map(roomId => getRoomMessages(targetUid, roomId))); + chatData = chatData.concat(_.flatten(result)); + }, { batch: 100 }); + + res.set('Content-Type', 'application/json') + .set('Content-Disposition', 'attachment; filename="' + targetUid + '_profile.json"') + .send({ + user: userData, + settings: userSettings, + ips: ips, + sessions: sessions, + usernames: usernames, + emails: emails, + messages: chatData, + bookmarks: bookmarks, + watchedTopics: watchedTopics, + upvoted: upvoted, + downvoted: downvoted, + following: followingData, + }); }; +async function getRoomMessages(uid, roomId) { + let data = []; + await batch.processSortedSet('uid:' + uid + ':chat:room:' + roomId + ':mids', async (mids) => { + const messageData = await db.getObjects(mids.map(mid => 'message:' + mid)); + data = data.concat(messageData.filter(m => m && m.fromuid === uid && !m.system) + .map(m => ({ content: m.content, timestamp: m.timestamp })) + ); + }, { batch: 500 }); + return data; +} + +async function getSetData(set, keyPrefix) { + let data = []; + await batch.processSortedSet(set, async (ids) => { + data = data.concat(await db.getObjects(ids.map(mid => keyPrefix + mid))); + }, { batch: 500 }); + return data; +} + + require('../promisify')(userController, [ 'getCurrentUser', 'getUserByUID', 'getUserByUsername', 'getUserByEmail', 'exportPosts', 'exportUploads', 'exportProfile',