From d05705183f3eb0cde6c3bce3a814a6a5c31117ed Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 22 Aug 2022 15:50:02 -0400 Subject: [PATCH] refactor: move export generation logic to v3 controller, GET/HEAD routes for exports re: #10384 --- public/src/client/account/consent.js | 17 ++++---- src/api/users.js | 35 +++++++++++++++++ src/controllers/user.js | 4 ++ src/controllers/write/users.js | 58 ++++++++++++++++++++++++++++ src/routes/write/users.js | 4 ++ src/socket.io/user/profile.js | 42 +++----------------- 6 files changed, 114 insertions(+), 46 deletions(-) diff --git a/public/src/client/account/consent.js b/public/src/client/account/consent.js index a43f29449b..41a3a0c77d 100644 --- a/public/src/client/account/consent.js +++ b/public/src/client/account/consent.js @@ -1,7 +1,7 @@ 'use strict'; -define('forum/account/consent', ['forum/account/header', 'alerts'], function (header, alerts) { +define('forum/account/consent', ['forum/account/header', 'alerts', 'api'], function (header, alerts, api) { const Consent = {}; Consent.init = function () { @@ -17,18 +17,15 @@ define('forum/account/consent', ['forum/account/header', 'alerts'], function (he }); }); - handleExport($('[data-action="export-profile"]'), 'user.exportProfile', '[[user:consent.export-profile-success]]'); - handleExport($('[data-action="export-posts"]'), 'user.exportPosts', '[[user:consent.export-posts-success]]'); - handleExport($('[data-action="export-uploads"]'), 'user.exportUploads', '[[user:consent.export-uploads-success]]'); + handleExport($('[data-action="export-profile"]'), 'profile', '[[user:consent.export-profile-success]]'); + handleExport($('[data-action="export-posts"]'), 'posts', '[[user:consent.export-posts-success]]'); + handleExport($('[data-action="export-uploads"]'), 'uploads', '[[user:consent.export-uploads-success]]'); - function handleExport(el, method, success) { + function handleExport(el, type, success) { el.on('click', function () { - socket.emit(method, { uid: ajaxify.data.uid }, function (err) { - if (err) { - return alerts.error(err); - } + api.post(`/users/${ajaxify.data.uid}/exports/${type}`).then(() => { alerts.success(success); - }); + }).catch(alerts.error); }); } }; diff --git a/src/api/users.js b/src/api/users.js index abc295acfd..c23f2c04d6 100644 --- a/src/api/users.js +++ b/src/api/users.js @@ -1,6 +1,7 @@ 'use strict'; const validator = require('validator'); +const winston = require('winston'); const db = require('../database'); const user = require('../user'); @@ -441,3 +442,37 @@ usersAPI.changePicture = async (caller, data) => { 'icon:bgColor': data.bgColor, }, ['picture', 'icon:bgColor']); }; + +usersAPI.generateExport = async (caller, { uid, type }) => { + const count = await db.incrObjectField('locks', `export:${uid}${type}`); + if (count > 1) { + throw new Error('[[error:already-exporting]]'); + } + + const child = require('child_process').fork(`./src/user/jobs/export-${type}.js`, [], { + env: process.env, + }); + child.send({ uid }); + child.on('error', async (err) => { + winston.error(err.stack); + await db.deleteObjectField('locks', `export:${uid}${type}`); + }); + child.on('exit', async () => { + await db.deleteObjectField('locks', `export:${uid}${type}`); + const userData = await user.getUserFields(uid, ['username', 'userslug']); + const { displayname } = userData; + const n = await notifications.create({ + bodyShort: `[[notifications:${type}-exported, ${displayname}]]`, + path: `/api/user/${userData.userslug}/export/${type}`, + nid: `${type}:export:${uid}`, + from: uid, + }); + await notifications.push(n, [caller.uid]); + await events.log({ + type: `export:${type}`, + uid: caller.uid, + targetUid: uid, + ip: caller.ip, + }); + }); +}; diff --git a/src/controllers/user.js b/src/controllers/user.js index 08772117a1..df71aa0b18 100644 --- a/src/controllers/user.js +++ b/src/controllers/user.js @@ -1,6 +1,7 @@ 'use strict'; const path = require('path'); +const winston = require('winston'); const user = require('../user'); const privileges = require('../privileges'); @@ -90,7 +91,10 @@ userController.exportProfile = async function (req, res, next) { sendExport(`${res.locals.uid}_profile.json`, 'application/json', res, next); }; +// DEPRECATED; Remove in NodeBB v3.0.0 function sendExport(filename, type, res, next) { + winston.warn(`[users/export] Access via page API is deprecated, use GET /api/v3/users/:uid/exports/:type instead.`); + res.sendFile(filename, { root: path.join(__dirname, '../../build/export'), headers: { diff --git a/src/controllers/write/users.js b/src/controllers/write/users.js index ddf1b5b93c..56d547aaaf 100644 --- a/src/controllers/write/users.js +++ b/src/controllers/write/users.js @@ -2,6 +2,9 @@ const util = require('util'); const nconf = require('nconf'); +const path = require('path'); +const crypto = require('crypto'); +const fs = require('fs').promises; const db = require('../../database'); const api = require('../../api'); @@ -15,6 +18,12 @@ const helpers = require('../helpers'); const Users = module.exports; +const exportMetadata = new Map([ + ['posts', ['csv', 'text/csv']], + ['uploads', ['zip', 'application/zip']], + ['profile', ['json', 'application/json']], +]); + const hasAdminPrivilege = async (uid, privilege) => { const ok = await privileges.admin.can(`admin:${privilege}`, uid); if (!ok) { @@ -296,3 +305,52 @@ Users.confirmEmail = async (req, res) => { helpers.formatApiResponse(404, res); } }; + +const prepareExport = async (req, res) => { + const [extension] = exportMetadata.get(req.params.type); + const filename = `${req.params.uid}_${req.params.type}.${extension}`; + try { + const stat = await fs.stat(path.join(__dirname, '../../../build/export', filename)); + const modified = new Date(stat.mtimeMs); + res.set('Last-Modified', modified.toUTCString()); + res.set('ETag', `"${crypto.createHash('md5').update(String(stat.mtimeMs)).digest('hex')}"`); + res.status(204); + return true; + } catch (e) { + res.status(404); + return false; + } +}; + +Users.checkExportByType = async (req, res) => { + await prepareExport(req, res); + res.end(); +}; + +Users.getExportByType = async (req, res) => { + const [extension, mime] = exportMetadata.get(req.params.type); + const filename = `${req.params.uid}_${req.params.type}.${extension}`; + + const exists = await prepareExport(req, res); + if (!exists) { + return res.end(); + } + + res.status(200); + res.sendFile(filename, { + root: path.join(__dirname, '../../../build/export'), + headers: { + 'Content-Type': mime, + 'Content-Disposition': `attachment; filename=${filename}`, + }, + }, (err) => { + if (err) { + throw err; + } + }); +}; + +Users.generateExportsByType = async (req, res) => { + await api.users.generateExport(req, req.params); + helpers.formatApiResponse(202, res); +}; diff --git a/src/routes/write/users.js b/src/routes/write/users.js index 6f22fa6166..980eccc8a2 100644 --- a/src/routes/write/users.js +++ b/src/routes/write/users.js @@ -51,6 +51,10 @@ function authenticatedRoutes() { setupApiRoute(router, 'get', '/:uid/emails/:email', [...middlewares, middleware.assert.user], controllers.write.users.getEmail); setupApiRoute(router, 'post', '/:uid/emails/:email/confirm', [...middlewares, middleware.assert.user], controllers.write.users.confirmEmail); + setupApiRoute(router, 'head', '/:uid/exports/:type', [...middlewares, middleware.assert.user, middleware.checkAccountPermissions], controllers.write.users.checkExportByType); + setupApiRoute(router, 'get', '/:uid/exports/:type', [...middlewares, middleware.assert.user, middleware.checkAccountPermissions], controllers.write.users.getExportByType); + setupApiRoute(router, 'post', '/:uid/exports/:type', [...middlewares, middleware.assert.user, middleware.checkAccountPermissions], controllers.write.users.generateExportsByType); + // Shorthand route to access user routes by userslug router.all('/+bySlug/:userslug*?', [], controllers.write.users.redirectBySlug); } diff --git a/src/socket.io/user/profile.js b/src/socket.io/user/profile.js index 2dcbb1af03..20a869056c 100644 --- a/src/socket.io/user/profile.js +++ b/src/socket.io/user/profile.js @@ -1,14 +1,12 @@ 'use strict'; -const winston = require('winston'); - const user = require('../../user'); -const events = require('../../events'); -const notifications = require('../../notifications'); const privileges = require('../../privileges'); -const db = require('../../database'); const plugins = require('../../plugins'); +const sockets = require('..'); +const api = require('../../api'); + module.exports = function (SocketUser) { SocketUser.updateCover = async function (socket, data) { if (!socket.uid) { @@ -64,6 +62,8 @@ module.exports = function (SocketUser) { }; async function doExport(socket, data, type) { + sockets.warnDeprecated(socket, 'POST /api/v3/users/:uid/exports/:type'); + if (!socket.uid) { throw new Error('[[error:invalid-uid]]'); } @@ -74,36 +74,6 @@ module.exports = function (SocketUser) { await user.isAdminOrSelf(socket.uid, data.uid); - const count = await db.incrObjectField('locks', `export:${data.uid}${type}`); - if (count > 1) { - throw new Error('[[error:already-exporting]]'); - } - - const child = require('child_process').fork(`./src/user/jobs/export-${type}.js`, [], { - env: process.env, - }); - child.send({ uid: data.uid }); - child.on('error', async (err) => { - winston.error(err.stack); - await db.deleteObjectField('locks', `export:${data.uid}${type}`); - }); - child.on('exit', async () => { - await db.deleteObjectField('locks', `export:${data.uid}${type}`); - const userData = await user.getUserFields(data.uid, ['username', 'userslug']); - const { displayname } = userData; - const n = await notifications.create({ - bodyShort: `[[notifications:${type}-exported, ${displayname}]]`, - path: `/api/user/${userData.userslug}/export/${type}`, - nid: `${type}:export:${data.uid}`, - from: data.uid, - }); - await notifications.push(n, [socket.uid]); - await events.log({ - type: `export:${type}`, - uid: socket.uid, - targetUid: data.uid, - ip: socket.ip, - }); - }); + api.users.generateExport(socket, { type, ...data }); } };