From a6564b7b4ed24f0d86e90486b4b16d31d89926c6 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 13 Apr 2018 11:58:31 -0400 Subject: [PATCH] endpoint to download user attachments, #6441 --- build/export/.gitignore | 1 + build/export/README | 5 +++ src/controllers/user.js | 86 +++++++++++++++++++++++++++++++++++++++++ src/events.js | 4 ++ src/routes/api.js | 1 + src/user/uploads.js | 15 +++++++ 6 files changed, 112 insertions(+) create mode 100644 build/export/README diff --git a/build/export/.gitignore b/build/export/.gitignore index 49ca10c54e..f8d55e0e3f 100644 --- a/build/export/.gitignore +++ b/build/export/.gitignore @@ -1,2 +1,3 @@ . !.gitignore +!README \ No newline at end of file diff --git a/build/export/README b/build/export/README new file mode 100644 index 0000000000..a9015033f4 --- /dev/null +++ b/build/export/README @@ -0,0 +1,5 @@ +This directory contains archives of user uploads that are prepared on-demand +when a user wants to retrieve a copy of their uploaded content. + +You can delete the files in here at will. They will just be regenerated if +requested again. \ No newline at end of file diff --git a/src/controllers/user.js b/src/controllers/user.js index edb7202b39..f1cc35e126 100644 --- a/src/controllers/user.js +++ b/src/controllers/user.js @@ -1,12 +1,17 @@ 'use strict'; var async = require('async'); +var path = require('path'); +var fs = require('fs'); +var winston = require('winston'); var converter = require('json-2-csv'); +var archiver = require('archiver'); var user = require('../user'); var meta = require('../meta'); var posts = require('../posts'); var batch = require('../batch'); +var events = require('../events'); var accountHelpers = require('./accounts/helpers'); var userController = module.exports; @@ -133,3 +138,84 @@ userController.exportPosts = function (req, res, next) { res.set('Content-Type', 'text/csv').set('Content-Disposition', 'attachment; filename="' + req.params.uid + '_posts.csv"').send(csv); }); }; + +userController.exportUploads = function (req, res, next) { + const archivePath = path.join(__dirname, '../../build/export', req.params.uid + '_uploads.zip'); + const archive = archiver('zip', { + zlib: { level: 9 }, // Sets the compression level. + }); + const maxAge = 1000 * 60 * 60 * 24; // 1 day + + const rootDirectory = path.join(__dirname, '../../public/uploads/'); + const trimPath = function (path) { + return path.replace(rootDirectory, ''); + }; + let isFresh = false; + const sendFile = function () { + events.log({ + type: 'export:uploads', + uid: req.uid, + targetUid: req.params.uid, + ip: req.ip, + fresh: isFresh, + }); + + res.sendFile(req.params.uid + '_uploads.zip', { + root: path.join(__dirname, '../../build/export'), + headers: { + 'Content-Disposition': 'attachment; filename=' + req.params.uid + '_uploads.zip', + maxAge: maxAge, + }, + }); + }; + + // Check for existing file, if exists and is < 1 day in age, send this instead + try { + fs.accessSync(archivePath, fs.constants.F_OK | fs.constants.R_OK); + isFresh = (Date.now() - fs.statSync(archivePath).mtimeMs) < maxAge; + if (isFresh) { + return sendFile(); + } + } catch (err) { + // File doesn't exist, continue + } + + const output = fs.createWriteStream(archivePath); + output.on('close', sendFile); + + archive.on('warning', function (err) { + switch (err.code) { + case 'ENOENT': + winston.warn('[user/export/uploads] File not found: ' + trimPath(err.path)); + break; + + default: + winston.warn('[user/export/uploads] Unexpected warning: ' + err.message); + break; + } + }); + + archive.on('error', function (err) { + switch (err.code) { + case 'EACCES': + winston.error('[user/export/uploads] File inaccessible: ' + trimPath(err.path)); + break; + + default: + winston.error('[user/export/uploads] Unable to construct archive: ' + err.message); + break; + } + + res.sendStatus(500); + }); + + archive.pipe(output); + winston.info('[user/export/uploads] Collating uploads for uid ' + req.params.uid); + user.collateUploads(req.params.uid, archive, function (err) { + if (err) { + return next(err); + } + + archive.finalize(); + }); +}; diff --git a/src/events.js b/src/events.js index c19a948579..cb8798ed70 100644 --- a/src/events.js +++ b/src/events.js @@ -12,6 +12,10 @@ var utils = require('./utils'); var events = module.exports; +/** + * Useful options in data: type, uid, ip, targetUid + * Everything else gets stringified and shown as pretty JSON string + */ events.log = function (data, callback) { callback = callback || function () {}; diff --git a/src/routes/api.js b/src/routes/api.js index 0e531b7236..cbd6972ef6 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -16,6 +16,7 @@ module.exports = function (app, middleware, controllers) { router.get('/user/email/:email', middleware.checkGlobalPrivacySettings, controllers.user.getUserByEmail); router.get('/user/uid/:uid/export/posts', middleware.checkAccountPermissions, controllers.user.exportPosts); + router.get('/user/uid/:uid/export/uploads', middleware.checkAccountPermissions, controllers.user.exportUploads); router.get('/:type/pid/:id', controllers.api.getObject); router.get('/:type/tid/:id', controllers.api.getObject); diff --git a/src/user/uploads.js b/src/user/uploads.js index faec37fce8..95b342ad27 100644 --- a/src/user/uploads.js +++ b/src/user/uploads.js @@ -6,6 +6,7 @@ var nconf = require('nconf'); var db = require('../database'); var file = require('../file'); +var batch = require('../batch'); module.exports = function (User) { User.deleteUpload = function (callerUid, uid, uploadName, callback) { @@ -32,4 +33,18 @@ module.exports = function (User) { }, ], callback); }; + + User.collateUploads = function (uid, archive, callback) { + batch.processSortedSet('uid:' + uid + ':uploads', function (files, next) { + files.forEach(function (file) { + archive.file(path.join(nconf.get('upload_path'), file), { + name: path.basename(file), + }); + }); + + setImmediate(next); + }, function (err) { + callback(err); + }); + }; };