feat: Configurable file storage locations (#886)

* feat: Make logfile location customizable

It may be desirable to log to a more standard location (e.g. in /var/log/),
or in some cases to turn logging to file off. To support these, use a
custom config property to determine the location of the output log file,
and default to the previous location if it is unset.

* feat: Support alternate storage locations for uploaded files

This involves a couple primary changes:
1) to make Sails' temporary file-upload directory a configurable location
   by using a common file-upload-receiving helper;
2) to create custom static routes for the file-upload locations, so they
   can be outside the application's public directory; and
3) to use the file-uploading handler everywhere that receives files, so
   config for the helper is applied to all file uploads consistently.

This is sufficient to allow the application directory to be deployed read-
only, with writable storage used for file uploads. The new config property
for Sails' temporary upload directory, combined with the existing settings
for user-avatar and background-image locations are sufficient to handle
uploads; the new custom routes handle serving those files from external
locations.

The default behavior of the application should be unchanged, with files
uploaded to, and served from, the public directory if the relevant
config properties aren't set to other values.

server/.env.sample

@@ -6,6 +6,8 @@ SECRET_KEY=notsecretkey
 
 ## Optional
 
+# LOG_FILE=
+
 # TRUST_PROXY=0
 # TOKEN_EXPIRES_IN=365 # In days
 

server/api/controllers/attachments/create.js

@@ -1,6 +1,3 @@
-const util = require('util');
-const { v4: uuid } = require('uuid');
-
 const Errors = {
   NOT_ENOUGH_RIGHTS: {
     notEnoughRights: 'Not enough rights',
@@ -61,16 +58,9 @@ module.exports = {
       throw Errors.NOT_ENOUGH_RIGHTS;
     }
 
-    const upload = util.promisify((options, callback) =>
-      this.req.file('file').upload(options, (error, files) => callback(error, files)),
-    );
-
     let files;
     try {
-      files = await upload({
-        saveAs: uuid(),
-        maxBytes: null,
-      });
+      files = await sails.helpers.utils.receiveFile('file', this.req);
     } catch (error) {
       return exits.uploadError(error.message); // TODO: add error
     }

server/api/controllers/boards/create.js

@@ -1,6 +1,3 @@
-const util = require('util');
-const { v4: uuid } = require('uuid');
-
 const Errors = {
   PROJECT_NOT_FOUND: {
     projectNotFound: 'Project not found',
@@ -69,16 +66,9 @@ module.exports = {
 
     let boardImport;
     if (inputs.importType && Object.values(Board.ImportTypes).includes(inputs.importType)) {
-      const upload = util.promisify((options, callback) =>
-        this.req.file('importFile').upload(options, (error, files) => callback(error, files)),
-      );
-
       let files;
       try {
-        files = await upload({
-          saveAs: uuid(),
-          maxBytes: null,
-        });
+        files = await sails.helpers.utils.receiveFile('importFile', this.req);
       } catch (error) {
         return exits.uploadError(error.message); // TODO: add error
       }

server/api/controllers/projects/update-background-image.js

@@ -1,6 +1,4 @@
-const util = require('util');
 const rimraf = require('rimraf');
-const { v4: uuid } = require('uuid');
 
 const Errors = {
   PROJECT_NOT_FOUND: {
@@ -53,16 +51,9 @@ module.exports = {
       throw Errors.PROJECT_NOT_FOUND; // Forbidden
     }
 
-    const upload = util.promisify((options, callback) =>
-      this.req.file('file').upload(options, (error, files) => callback(error, files)),
-    );
-
     let files;
     try {
-      files = await upload({
-        saveAs: uuid(),
-        maxBytes: null,
-      });
+      files = await sails.helpers.utils.receiveFile('file', this.req);
     } catch (error) {
       return exits.uploadError(error.message); // TODO: add error
     }

server/api/controllers/users/update-avatar.js

@@ -1,6 +1,4 @@
-const util = require('util');
 const rimraf = require('rimraf');
-const { v4: uuid } = require('uuid');
 
 const Errors = {
   USER_NOT_FOUND: {
@@ -54,16 +52,9 @@ module.exports = {
       user = currentUser;
     }
 
-    const upload = util.promisify((options, callback) =>
-      this.req.file('file').upload(options, (error, files) => callback(error, files)),
-    );
-
     let files;
     try {
-      files = await upload({
-        saveAs: uuid(),
-        maxBytes: null,
-      });
+      files = await sails.helpers.utils.receiveFile('file', this.req);
     } catch (error) {
       return exits.uploadError(error.message); // TODO: add error
     }

server/api/helpers/utils/receive-file.js

@@ -0,0 +1,41 @@
+const util = require('util');
+const { v4: uuid } = require('uuid');
+
+async function doUpload(paramName, req, options) {
+  const uploadOptions = {
+    ...options,
+    dirname: options.dirname || sails.config.custom.fileUploadTmpDir,
+  };
+  const upload = util.promisify((opts, callback) => {
+    return req.file(paramName).upload(opts, (error, files) => callback(error, files));
+  });
+  return upload(uploadOptions);
+}
+
+module.exports = {
+  friendlyName: 'Receive uploaded file from request',
+  description:
+    "Store a file uploaded from a MIME-multipart request part. The request part name must be 'file'; the resulting file will have a unique UUID-based name with the same extension.",
+  inputs: {
+    paramName: {
+      type: 'string',
+      required: true,
+      description: 'The MIME multi-part parameter containing the file to receive.',
+    },
+    req: {
+      type: 'ref',
+      required: true,
+      description: 'The request to receive the file from.',
+    },
+  },
+
+  fn: async function modFn(inputs, exits) {
+    exits.success(
+      await doUpload(inputs.paramName, inputs.req, {
+        saveAs: uuid(),
+        dirname: sails.config.custom.fileUploadTmpDir,
+        maxBytes: null,
+      }),
+    );
+  },
+};

server/config/custom.js

@@ -27,6 +27,9 @@ module.exports.custom = {
 
   tokenExpiresIn: parseInt(process.env.TOKEN_EXPIRES_IN, 10) || 365,
 
+  // Location to receive uploaded files in. Default (non-string value) is a Sails-specific location.
+  fileUploadTmpDir: null,
+
   userAvatarsPath: path.join(sails.config.paths.public, 'user-avatars'),
   userAvatarsUrl: `${process.env.BASE_URL}/user-avatars`,
 

server/config/routes.js

@@ -1,3 +1,56 @@
+const serveStatic = require('serve-static');
+const sails = require('sails');
+const path = require('path');
+
+// Remove prefix from urlPath, assuming completely matches a subpath of
+// urlPath. The result preserves query params and fragment if present
+//
+// Examples:
+// '/foo', '/foo/bar'     -> '/bar'
+// '/foo', '/foo'         -> '/'
+// '/foo', '/foo?baz=bux' -> '/?baz=bux'
+// '/foo', '/foobar'      -> '/foobar'
+function removeRoutePrefix(prefix, urlPath) {
+  if (urlPath.startsWith(prefix)) {
+    const subpath = urlPath.substring(prefix.length);
+    if (subpath.startsWith('/')) {
+      // Prefix matched a complete set of path segments, with a valid path
+      // remaining.
+      return subpath;
+    }
+
+    if (subpath.length === 0 || subpath.startsWith('?') || subpath.startsWith('#')) {
+      // Prefix matched a complete set of path segments, but there is no path
+      // remaining. Add '/'.
+      return `/${subpath}`;
+    }
+  }
+
+  // Either the prefix didn't match at all, or it wasn't a complete path match
+  // (e.g. we don't want to treat '/foo' as a prefix of '/foobar'). Leave the
+  // path as-is.
+  return urlPath;
+}
+
+function staticDirServer(prefix, dirFn) {
+  return function handleReq(req, res, next) {
+    // Custom config properties are not available when the routes config is
+    // loaded, so resolve the target value just before serving the request.
+    const dir = dirFn();
+    const staticServer = serveStatic(dir, { index: false });
+
+    const reqPath = req.url;
+    if (reqPath.startsWith(prefix)) {
+      // serve-static treats the request url as a sub-path of
+      // static root; remove the leading route prefix so the static root
+      // doesn't have to include the prefix as a subdirectory.
+      req.url = removeRoutePrefix(prefix, req.url);
+      return staticServer(req, res, next);
+    }
+    return next();
+  };
+}
+
 /**
  * Route Mappings
  * (sails.config.routes)
@@ -81,6 +134,18 @@ module.exports.routes = {
   'GET /api/notifications/:id': 'notifications/show',
   'PATCH /api/notifications/:ids': 'notifications/update',
 
+  'GET /user-avatars/*': {
+    fn: staticDirServer('/user-avatars', () => path.resolve(sails.config.custom.userAvatarsPath)),
+    skipAssets: false,
+  },
+
+  'GET /project-background-images/*': {
+    fn: staticDirServer('/project-background-images', () =>
+      path.resolve(sails.config.custom.projectBackgroundImagesPath),
+    ),
+    skipAssets: false,
+  },
+
   'GET /attachments/:id/download/:filename': {
     action: 'attachments/download',
     skipAssets: false,

server/package-lock.json

@@ -24,6 +24,7 @@
         "sails-hook-orm": "^4.0.3",
         "bcrypt": "^5.1.1",
         "validator": "^13.12.0",
+        "serve-static": "^1.13.1",
         "sails-hook-sockets": "^3.0.1",
         "openid-client": "^5.7.0",
         "rimraf": "^5.0.10",
@@ -844,6 +845,29 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/sails/node_modules/express/node_modules/serve-static/node_modules/send": {
+      "version": "0.19.0",
+      "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
+      "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==",
+      "dependencies": {
+        "debug": "2.6.9",
+        "depd": "2.0.0",
+        "destroy": "1.2.0",
+        "encodeurl": "~1.0.2",
+        "escape-html": "~1.0.3",
+        "etag": "~1.8.1",
+        "fresh": "0.5.2",
+        "http-errors": "2.0.0",
+        "mime": "1.6.0",
+        "ms": "2.1.3",
+        "on-finished": "2.4.1",
+        "range-parser": "~1.2.1",
+        "statuses": "2.0.1"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
     "node_modules/serve-favicon": {
       "version": "2.4.5",
       "resolved": "https://registry.npmjs.org/serve-favicon/-/serve-favicon-2.4.5.tgz",
@@ -1716,6 +1740,14 @@
         "color-string": "^1.6.0"
       }
     },
+    "node_modules/whelk/node_modules/camelcase": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz",
+      "integrity": "sha512-wzLkDa4K/mzI1OSITC+DUyjgIl/ETNHE9QvYgy6J6Jvqyyz4C0Xfd+lQhb19sX2jMpZV4IssUn0VDVmglV+s4g==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/balanced-match": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
@@ -3959,6 +3991,7 @@
       "version": "6.3.0",
       "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
       "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
+      "dev": true,
       "engines": {
         "node": ">=10"
       },

server/package.json

@@ -43,6 +43,7 @@
     "sails-hook-orm": "^4.0.3",
     "sails-hook-sockets": "^3.0.1",
     "sails-postgresql": "^5.0.1",
+    "serve-static": "^1.13.1",
     "sharp": "^0.33.5",
     "stream-to-array": "^2.3.0",
     "uuid": "^9.0.1",

server/utils/logger.js

@@ -6,7 +6,8 @@ const winston = require('winston');
  */
 const defaultLogTimestampFormat = 'YYYY-MM-DD HH:mm:ss';
 
-const logfile = `${process.cwd()}/logs/planka.log`;
+const logfile =
+  'LOG_FILE' in process.env ? process.env.LOG_FILE : `${process.cwd()}/logs/planka.log`;
 
 /**
  * Log level for both console and file log sinks.