From 37fc7847e8de0e4ed7ff00f5ff2fadd8c509dd35 Mon Sep 17 00:00:00 2001 From: Matthew Stickney Date: Fri, 20 Sep 2024 14:29:11 -0400 Subject: [PATCH] feat: Configurable file storage locations (#886) * feat: Make logfile location customizable It may be desirable to log to a more standard location (e.g. in /var/log/), or in some cases to turn logging to file off. To support these, use a custom config property to determine the location of the output log file, and default to the previous location if it is unset. * feat: Support alternate storage locations for uploaded files This involves a couple primary changes: 1) to make Sails' temporary file-upload directory a configurable location by using a common file-upload-receiving helper; 2) to create custom static routes for the file-upload locations, so they can be outside the application's public directory; and 3) to use the file-uploading handler everywhere that receives files, so config for the helper is applied to all file uploads consistently. This is sufficient to allow the application directory to be deployed read- only, with writable storage used for file uploads. The new config property for Sails' temporary upload directory, combined with the existing settings for user-avatar and background-image locations are sufficient to handle uploads; the new custom routes handle serving those files from external locations. The default behavior of the application should be unchanged, with files uploaded to, and served from, the public directory if the relevant config properties aren't set to other values. --- server/.env.sample | 2 + server/api/controllers/attachments/create.js | 12 +--- server/api/controllers/boards/create.js | 12 +--- .../projects/update-background-image.js | 11 +--- server/api/controllers/users/update-avatar.js | 11 +--- server/api/helpers/utils/receive-file.js | 41 ++++++++++++ server/config/custom.js | 3 + server/config/routes.js | 65 +++++++++++++++++++ server/package-lock.json | 33 ++++++++++ server/package.json | 1 + server/utils/logger.js | 3 +- 11 files changed, 151 insertions(+), 43 deletions(-) create mode 100644 server/api/helpers/utils/receive-file.js diff --git a/server/.env.sample b/server/.env.sample index 46442b40..28be92ff 100644 --- a/server/.env.sample +++ b/server/.env.sample @@ -6,6 +6,8 @@ SECRET_KEY=notsecretkey ## Optional +# LOG_FILE= + # TRUST_PROXY=0 # TOKEN_EXPIRES_IN=365 # In days diff --git a/server/api/controllers/attachments/create.js b/server/api/controllers/attachments/create.js index 03e0c8cf..387368dc 100644 --- a/server/api/controllers/attachments/create.js +++ b/server/api/controllers/attachments/create.js @@ -1,6 +1,3 @@ -const util = require('util'); -const { v4: uuid } = require('uuid'); - const Errors = { NOT_ENOUGH_RIGHTS: { notEnoughRights: 'Not enough rights', @@ -61,16 +58,9 @@ module.exports = { throw Errors.NOT_ENOUGH_RIGHTS; } - const upload = util.promisify((options, callback) => - this.req.file('file').upload(options, (error, files) => callback(error, files)), - ); - let files; try { - files = await upload({ - saveAs: uuid(), - maxBytes: null, - }); + files = await sails.helpers.utils.receiveFile('file', this.req); } catch (error) { return exits.uploadError(error.message); // TODO: add error } diff --git a/server/api/controllers/boards/create.js b/server/api/controllers/boards/create.js index 990cb85a..e1e41d58 100755 --- a/server/api/controllers/boards/create.js +++ b/server/api/controllers/boards/create.js @@ -1,6 +1,3 @@ -const util = require('util'); -const { v4: uuid } = require('uuid'); - const Errors = { PROJECT_NOT_FOUND: { projectNotFound: 'Project not found', @@ -69,16 +66,9 @@ module.exports = { let boardImport; if (inputs.importType && Object.values(Board.ImportTypes).includes(inputs.importType)) { - const upload = util.promisify((options, callback) => - this.req.file('importFile').upload(options, (error, files) => callback(error, files)), - ); - let files; try { - files = await upload({ - saveAs: uuid(), - maxBytes: null, - }); + files = await sails.helpers.utils.receiveFile('importFile', this.req); } catch (error) { return exits.uploadError(error.message); // TODO: add error } diff --git a/server/api/controllers/projects/update-background-image.js b/server/api/controllers/projects/update-background-image.js index 2045ee1e..b09f355b 100755 --- a/server/api/controllers/projects/update-background-image.js +++ b/server/api/controllers/projects/update-background-image.js @@ -1,6 +1,4 @@ -const util = require('util'); const rimraf = require('rimraf'); -const { v4: uuid } = require('uuid'); const Errors = { PROJECT_NOT_FOUND: { @@ -53,16 +51,9 @@ module.exports = { throw Errors.PROJECT_NOT_FOUND; // Forbidden } - const upload = util.promisify((options, callback) => - this.req.file('file').upload(options, (error, files) => callback(error, files)), - ); - let files; try { - files = await upload({ - saveAs: uuid(), - maxBytes: null, - }); + files = await sails.helpers.utils.receiveFile('file', this.req); } catch (error) { return exits.uploadError(error.message); // TODO: add error } diff --git a/server/api/controllers/users/update-avatar.js b/server/api/controllers/users/update-avatar.js index fc628564..fbd23b9f 100755 --- a/server/api/controllers/users/update-avatar.js +++ b/server/api/controllers/users/update-avatar.js @@ -1,6 +1,4 @@ -const util = require('util'); const rimraf = require('rimraf'); -const { v4: uuid } = require('uuid'); const Errors = { USER_NOT_FOUND: { @@ -54,16 +52,9 @@ module.exports = { user = currentUser; } - const upload = util.promisify((options, callback) => - this.req.file('file').upload(options, (error, files) => callback(error, files)), - ); - let files; try { - files = await upload({ - saveAs: uuid(), - maxBytes: null, - }); + files = await sails.helpers.utils.receiveFile('file', this.req); } catch (error) { return exits.uploadError(error.message); // TODO: add error } diff --git a/server/api/helpers/utils/receive-file.js b/server/api/helpers/utils/receive-file.js new file mode 100644 index 00000000..84319298 --- /dev/null +++ b/server/api/helpers/utils/receive-file.js @@ -0,0 +1,41 @@ +const util = require('util'); +const { v4: uuid } = require('uuid'); + +async function doUpload(paramName, req, options) { + const uploadOptions = { + ...options, + dirname: options.dirname || sails.config.custom.fileUploadTmpDir, + }; + const upload = util.promisify((opts, callback) => { + return req.file(paramName).upload(opts, (error, files) => callback(error, files)); + }); + return upload(uploadOptions); +} + +module.exports = { + friendlyName: 'Receive uploaded file from request', + description: + "Store a file uploaded from a MIME-multipart request part. The request part name must be 'file'; the resulting file will have a unique UUID-based name with the same extension.", + inputs: { + paramName: { + type: 'string', + required: true, + description: 'The MIME multi-part parameter containing the file to receive.', + }, + req: { + type: 'ref', + required: true, + description: 'The request to receive the file from.', + }, + }, + + fn: async function modFn(inputs, exits) { + exits.success( + await doUpload(inputs.paramName, inputs.req, { + saveAs: uuid(), + dirname: sails.config.custom.fileUploadTmpDir, + maxBytes: null, + }), + ); + }, +}; diff --git a/server/config/custom.js b/server/config/custom.js index 770695b7..44a72a14 100644 --- a/server/config/custom.js +++ b/server/config/custom.js @@ -27,6 +27,9 @@ module.exports.custom = { tokenExpiresIn: parseInt(process.env.TOKEN_EXPIRES_IN, 10) || 365, + // Location to receive uploaded files in. Default (non-string value) is a Sails-specific location. + fileUploadTmpDir: null, + userAvatarsPath: path.join(sails.config.paths.public, 'user-avatars'), userAvatarsUrl: `${process.env.BASE_URL}/user-avatars`, diff --git a/server/config/routes.js b/server/config/routes.js index e1961c9f..e23924f9 100644 --- a/server/config/routes.js +++ b/server/config/routes.js @@ -1,3 +1,56 @@ +const serveStatic = require('serve-static'); +const sails = require('sails'); +const path = require('path'); + +// Remove prefix from urlPath, assuming completely matches a subpath of +// urlPath. The result preserves query params and fragment if present +// +// Examples: +// '/foo', '/foo/bar' -> '/bar' +// '/foo', '/foo' -> '/' +// '/foo', '/foo?baz=bux' -> '/?baz=bux' +// '/foo', '/foobar' -> '/foobar' +function removeRoutePrefix(prefix, urlPath) { + if (urlPath.startsWith(prefix)) { + const subpath = urlPath.substring(prefix.length); + if (subpath.startsWith('/')) { + // Prefix matched a complete set of path segments, with a valid path + // remaining. + return subpath; + } + + if (subpath.length === 0 || subpath.startsWith('?') || subpath.startsWith('#')) { + // Prefix matched a complete set of path segments, but there is no path + // remaining. Add '/'. + return `/${subpath}`; + } + } + + // Either the prefix didn't match at all, or it wasn't a complete path match + // (e.g. we don't want to treat '/foo' as a prefix of '/foobar'). Leave the + // path as-is. + return urlPath; +} + +function staticDirServer(prefix, dirFn) { + return function handleReq(req, res, next) { + // Custom config properties are not available when the routes config is + // loaded, so resolve the target value just before serving the request. + const dir = dirFn(); + const staticServer = serveStatic(dir, { index: false }); + + const reqPath = req.url; + if (reqPath.startsWith(prefix)) { + // serve-static treats the request url as a sub-path of + // static root; remove the leading route prefix so the static root + // doesn't have to include the prefix as a subdirectory. + req.url = removeRoutePrefix(prefix, req.url); + return staticServer(req, res, next); + } + return next(); + }; +} + /** * Route Mappings * (sails.config.routes) @@ -81,6 +134,18 @@ module.exports.routes = { 'GET /api/notifications/:id': 'notifications/show', 'PATCH /api/notifications/:ids': 'notifications/update', + 'GET /user-avatars/*': { + fn: staticDirServer('/user-avatars', () => path.resolve(sails.config.custom.userAvatarsPath)), + skipAssets: false, + }, + + 'GET /project-background-images/*': { + fn: staticDirServer('/project-background-images', () => + path.resolve(sails.config.custom.projectBackgroundImagesPath), + ), + skipAssets: false, + }, + 'GET /attachments/:id/download/:filename': { action: 'attachments/download', skipAssets: false, diff --git a/server/package-lock.json b/server/package-lock.json index 3ad71f96..9a4e1d9d 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -24,6 +24,7 @@ "sails-hook-orm": "^4.0.3", "bcrypt": "^5.1.1", "validator": "^13.12.0", + "serve-static": "^1.13.1", "sails-hook-sockets": "^3.0.1", "openid-client": "^5.7.0", "rimraf": "^5.0.10", @@ -844,6 +845,29 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sails/node_modules/express/node_modules/serve-static/node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/serve-favicon": { "version": "2.4.5", "resolved": "https://registry.npmjs.org/serve-favicon/-/serve-favicon-2.4.5.tgz", @@ -1716,6 +1740,14 @@ "color-string": "^1.6.0" } }, + "node_modules/whelk/node_modules/camelcase": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", + "integrity": "sha512-wzLkDa4K/mzI1OSITC+DUyjgIl/ETNHE9QvYgy6J6Jvqyyz4C0Xfd+lQhb19sX2jMpZV4IssUn0VDVmglV+s4g==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -3959,6 +3991,7 @@ "version": "6.3.0", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, "engines": { "node": ">=10" }, diff --git a/server/package.json b/server/package.json index 44381c04..3b7b4c27 100644 --- a/server/package.json +++ b/server/package.json @@ -43,6 +43,7 @@ "sails-hook-orm": "^4.0.3", "sails-hook-sockets": "^3.0.1", "sails-postgresql": "^5.0.1", + "serve-static": "^1.13.1", "sharp": "^0.33.5", "stream-to-array": "^2.3.0", "uuid": "^9.0.1", diff --git a/server/utils/logger.js b/server/utils/logger.js index d1c3cc70..808222d1 100644 --- a/server/utils/logger.js +++ b/server/utils/logger.js @@ -6,7 +6,8 @@ const winston = require('winston'); */ const defaultLogTimestampFormat = 'YYYY-MM-DD HH:mm:ss'; -const logfile = `${process.cwd()}/logs/planka.log`; +const logfile = + 'LOG_FILE' in process.env ? process.env.LOG_FILE : `${process.cwd()}/logs/planka.log`; /** * Log level for both console and file log sinks.