fix: Secure S3 attachments, bump SDK, refactoring

Closes #673

docker-compose-dev.yml

@@ -25,9 +25,15 @@ services:
       # - KNEX_REJECT_UNAUTHORIZED_SSL_CERTIFICATE=false
 
       # - SHOW_DETAILED_AUTH_ERRORS=false # Set to true to show more detailed authentication error messages. It should not be enabled without a rate limiter for security reasons.
-
       # - ALLOW_ALL_TO_CREATE_PROJECTS=true
 
+      # - S3_ENDPOINT=
+      # - S3_REGION=
+      # - S3_ACCESS_KEY_ID=
+      # - S3_SECRET_ACCESS_KEY=
+      # - S3_BUCKET=
+      # - S3_FORCE_PATH_STYLE=true
+
       # - OIDC_ISSUER=
       # - OIDC_CLIENT_ID=
       # - OIDC_CLIENT_SECRET=

docker-compose.yml

@@ -32,9 +32,15 @@ services:
       # - DEFAULT_ADMIN_USERNAME=demo
 
       # - SHOW_DETAILED_AUTH_ERRORS=false # Set to true to show more detailed authentication error messages. It should not be enabled without a rate limiter for security reasons.
-
       # - ALLOW_ALL_TO_CREATE_PROJECTS=true
 
+      # - S3_ENDPOINT=
+      # - S3_REGION=
+      # - S3_ACCESS_KEY_ID=
+      # - S3_SECRET_ACCESS_KEY=
+      # - S3_BUCKET=
+      # - S3_FORCE_PATH_STYLE=true
+
       # - OIDC_ISSUER=
       # - OIDC_CLIENT_ID=
       # - OIDC_CLIENT_SECRET=
@@ -80,14 +86,6 @@ services:
       # - TELEGRAM_BOT_TOKEN=
       # - TELEGRAM_CHAT_ID=
       # - TELEGRAM_THREAD_ID=
-
-      # Attachments S3
-      # - S3_ENABLE=true
-      # - S3_REGION=
-      # - S3_ENDPOINT=
-      # - S3_BUCKET=
-      # - S3_ACCESS_KEY=
-      # - S3_SECRET_KEY=
     depends_on:
       postgres:
         condition: service_healthy
@@ -101,7 +99,7 @@ services:
       - POSTGRES_DB=planka
       - POSTGRES_HOST_AUTH_METHOD=trust
     healthcheck:
-      test: ['CMD-SHELL', 'pg_isready -U postgres -d planka']
+      test: ["CMD-SHELL", "pg_isready -U postgres -d planka"]
       interval: 10s
       timeout: 5s
       retries: 5

server/.env.sample

@@ -25,9 +25,15 @@ SECRET_KEY=notsecretkey
 # DEFAULT_ADMIN_USERNAME=demo
 
 # SHOW_DETAILED_AUTH_ERRORS=false # Set to true to show more detailed authentication error messages. It should not be enabled without a rate limiter for security reasons.
-
 # ALLOW_ALL_TO_CREATE_PROJECTS=true
 
+# S3_ENDPOINT=
+# S3_REGION=
+# S3_ACCESS_KEY_ID=
+# S3_SECRET_ACCESS_KEY=
+# S3_BUCKET=
+# S3_FORCE_PATH_STYLE=true
+
 # OIDC_ISSUER=
 # OIDC_CLIENT_ID=
 # OIDC_CLIENT_SECRET=
@@ -75,10 +81,3 @@ SECRET_KEY=notsecretkey
 ## Do not edit this
 
 TZ=UTC
-
-# S3_ENABLE=true
-# S3_REGION=
-# S3_ENDPOINT=
-# S3_BUCKET=
-# S3_ACCESS_KEY=
-# S3_SECRET_KEY=

server/api/controllers/attachments/download-thumbnail.js

@@ -1,6 +1,3 @@
-const fs = require('fs');
-const path = require('path');
-
 const Errors = {
   ATTACHMENT_NOT_FOUND: {
     attachmentNotFound: 'Attachment not found',
@@ -46,20 +43,20 @@ module.exports = {
       throw Errors.ATTACHMENT_NOT_FOUND;
     }
 
-    const filePath = path.join(
+    const fileManager = sails.hooks['file-manager'].getInstance();
-      sails.config.custom.attachmentsPath,
-      attachment.dirname,
-      'thumbnails',
-      `cover-256.${attachment.image.thumbnailsExtension}`,
-    );
 
-    if (!fs.existsSync(filePath)) {
+    let readStream;
+    try {
+      readStream = await fileManager.read(
+        `${sails.config.custom.attachmentsPathSegment}/${attachment.dirname}/thumbnails/cover-256.${attachment.image.thumbnailsExtension}`,
+      );
+    } catch (error) {
       throw Errors.ATTACHMENT_NOT_FOUND;
     }
 
     this.res.type('image/jpeg');
     this.res.set('Cache-Control', 'private, max-age=900'); // TODO: move to config
 
-    return exits.success(fs.createReadStream(filePath));
+    return exits.success(readStream);
   },
 };

server/api/controllers/attachments/download.js

@@ -1,4 +1,3 @@
-const fs = require('fs');
 const path = require('path');
 
 const Errors = {
@@ -42,13 +41,14 @@ module.exports = {
       }
     }
 
-    const filePath = path.join(
+    const fileManager = sails.hooks['file-manager'].getInstance();
-      sails.config.custom.attachmentsPath,
-      attachment.dirname,
-      attachment.filename,
-    );
 
-    if (!fs.existsSync(filePath)) {
+    let readStream;
+    try {
+      readStream = await fileManager.read(
+        `${sails.config.custom.attachmentsPathSegment}/${attachment.dirname}/${attachment.filename}`,
+      );
+    } catch (error) {
       throw Errors.ATTACHMENT_NOT_FOUND;
     }
 
@@ -58,6 +58,6 @@ module.exports = {
     }
     this.res.set('Cache-Control', 'private, max-age=900'); // TODO: move to config
 
-    return exits.success(fs.createReadStream(filePath));
+    return exits.success(readStream);
   },
 };

server/api/helpers/attachments/delete-one.js

@@ -1,6 +1,3 @@
-const path = require('path');
-const rimraf = require('rimraf');
-
 module.exports = {
   inputs: {
     record: {
@@ -50,26 +47,12 @@ module.exports = {
     const attachment = await Attachment.archiveOne(inputs.record.id);
 
     if (attachment) {
+      const fileManager = sails.hooks['file-manager'].getInstance();
+
       try {
-        const type = attachment.type || 'local';
+        await fileManager.deleteFolder(
-        if (type === 's3') {
+          `${sails.config.custom.attachmentsPathSegment}/${attachment.dirname}`,
-          const client = await sails.helpers.utils.getSimpleStorageServiceClient();
+        );
-          if (client) {
-            if (attachment.url) {
-              const parsedUrl = new URL(attachment.url);
-              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
-            }
-            if (attachment.thumb) {
-              const parsedUrl = new URL(attachment.thumb);
-              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
-            }
-          }
-        }
-      } catch (error) {
-        console.warn(error.stack); // eslint-disable-line no-console
-      }
-      try {
-        rimraf.sync(path.join(sails.config.custom.attachmentsPath, attachment.dirname));
       } catch (error) {
         console.warn(error.stack); // eslint-disable-line no-console
       }

server/api/helpers/attachments/process-uploaded-file.js

@@ -1,7 +1,4 @@
-const fs = require('fs');
+const { rimraf } = require('rimraf');
-const path = require('path');
-const rimraf = require('rimraf');
-const moveFile = require('move-file');
 const { v4: uuid } = require('uuid');
 const sharp = require('sharp');
 
@@ -16,86 +13,19 @@ module.exports = {
   },
 
   async fn(inputs) {
-    const dirname = uuid();
+    const fileManager = sails.hooks['file-manager'].getInstance();
 
+    const dirname = uuid();
+    const folderPathSegment = `${sails.config.custom.attachmentsPathSegment}/${dirname}`;
     const filename = filenamify(inputs.file.filename);
 
-    const rootPath = path.join(sails.config.custom.attachmentsPath, dirname);
+    const filePath = await fileManager.move(
-    const filePath = path.join(rootPath, filename);
+      inputs.file.fd,
+      `${folderPathSegment}/${filename}`,
+      inputs.file.type,
+    );
 
-    if (sails.config.custom.s3Config) {
+    let image = sharp(filePath || inputs.file.fd, {
-      const client = await sails.helpers.utils.getSimpleStorageServiceClient();
-      const s3Image = await client.upload({
-        Body: fs.createReadStream(inputs.file.fd),
-        Key: `attachments/${dirname}/${filename}`,
-        ContentType: inputs.file.type,
-      });
-
-      let image = sharp(inputs.file.fd, {
-        animated: true,
-      });
-
-      let metadata;
-      try {
-        metadata = await image.metadata();
-      } catch (error) {} // eslint-disable-line no-empty
-
-      const fileData = {
-        type: 's3',
-        dirname,
-        filename,
-        thumb: null,
-        image: null,
-        url: s3Image.Location,
-        name: inputs.file.filename,
-      };
-
-      if (metadata && !['svg', 'pdf'].includes(metadata.format)) {
-        let { width, pageHeight: height = metadata.height } = metadata;
-        if (metadata.orientation && metadata.orientation > 4) {
-          [image, width, height] = [image.rotate(), height, width];
-        }
-
-        const isPortrait = height > width;
-        const thumbnailsExtension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
-
-        try {
-          const resizeBuffer = await image
-            .resize(
-              256,
-              isPortrait ? 320 : undefined,
-              width < 256 || (isPortrait && height < 320)
-                ? {
-                    kernel: sharp.kernel.nearest,
-                  }
-                : undefined,
-            )
-            .toBuffer();
-          const s3Thumb = await client.upload({
-            Key: `attachments/${dirname}/thumbnails/cover-256.${thumbnailsExtension}`,
-            Body: resizeBuffer,
-            ContentType: inputs.file.type,
-          });
-          fileData.thumb = s3Thumb.Location;
-          fileData.image = { width, height };
-        } catch (error1) {
-          console.warn(error2.stack); // eslint-disable-line no-console
-        }
-      }
-
-      try {
-        rimraf.sync(inputs.file.fd);
-      } catch (error) {
-        console.warn(error.stack); // eslint-disable-line no-console
-      }
-
-      return fileData;
-    }
-
-    fs.mkdirSync(rootPath);
-    await moveFile(inputs.file.fd, filePath);
-
-    let image = sharp(filePath, {
       animated: true,
     });
 
@@ -105,7 +35,6 @@ module.exports = {
     } catch (error) {} // eslint-disable-line no-empty
 
     const fileData = {
-      type: 'local',
       dirname,
       filename,
       image: null,
@@ -113,9 +42,6 @@ module.exports = {
     };
 
     if (metadata && !['svg', 'pdf'].includes(metadata.format)) {
-      const thumbnailsPath = path.join(rootPath, 'thumbnails');
-      fs.mkdirSync(thumbnailsPath);
-
       let { width, pageHeight: height = metadata.height } = metadata;
       if (metadata.orientation && metadata.orientation > 4) {
         [image, width, height] = [image.rotate(), height, width];
@@ -125,7 +51,7 @@ module.exports = {
       const thumbnailsExtension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
 
       try {
-        await image
+        const resizeBuffer = await image
           .resize(
             256,
             isPortrait ? 320 : undefined,
@@ -135,19 +61,29 @@ module.exports = {
                 }
               : undefined,
           )
-          .toFile(path.join(thumbnailsPath, `cover-256.${thumbnailsExtension}`));
+          .toBuffer();
+
+        await fileManager.save(
+          `${folderPathSegment}/thumbnails/cover-256.${thumbnailsExtension}`,
+          resizeBuffer,
+          inputs.file.type,
+        );
 
         fileData.image = {
           width,
           height,
           thumbnailsExtension,
         };
-      } catch (error1) {
+      } catch (error) {
-        try {
+        console.warn(error.stack); // eslint-disable-line no-console
-          rimraf.sync(thumbnailsPath);
+      }
-        } catch (error2) {
+    }
-          console.warn(error2.stack); // eslint-disable-line no-console
+
-        }
+    if (!filePath) {
+      try {
+        await rimraf(inputs.file.fd);
+      } catch (error) {
+        console.warn(error.stack); // eslint-disable-line no-console
       }
     }
 

server/api/helpers/boards/process-uploaded-trello-import-file.js

@@ -1,5 +1,5 @@
-const fs = require('fs').promises;
+const fs = require('fs');
-const rimraf = require('rimraf');
+const { rimraf } = require('rimraf');
 
 module.exports = {
   inputs: {
@@ -14,7 +14,7 @@ module.exports = {
   },
 
   async fn(inputs) {
-    const content = await fs.readFile(inputs.file.fd);
+    const content = await fs.promises.readFile(inputs.file.fd);
     const trelloBoard = JSON.parse(content);
 
     if (
@@ -28,7 +28,7 @@ module.exports = {
     }
 
     try {
-      rimraf.sync(inputs.file.fd);
+      await rimraf(inputs.file.fd);
     } catch (error) {
       console.warn(error.stack); // eslint-disable-line no-console
     }

server/api/helpers/projects/process-uploaded-background-image-file.js

@@ -1,6 +1,4 @@
-const fs = require('fs');
+const { rimraf } = require('rimraf');
-const path = require('path');
-const rimraf = require('rimraf');
 const { v4: uuid } = require('uuid');
 const sharp = require('sharp');
 
@@ -32,7 +30,10 @@ module.exports = {
       throw 'fileIsNotImage';
     }
 
+    const fileManager = sails.hooks['file-manager'].getInstance();
+
     const dirname = uuid();
+    const folderPathSegment = `${sails.config.custom.projectBackgroundImagesPathSegment}/${dirname}`;
 
     let { width, pageHeight: height = metadata.height } = metadata;
     if (metadata.orientation && metadata.orientation > 4) {
@@ -41,68 +42,16 @@ module.exports = {
 
     const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
 
-    if (sails.config.custom.s3Config) {
-      const client = await sails.helpers.utils.getSimpleStorageServiceClient();
-      let originalUrl = '';
-      let thumbUrl = '';
-
-      try {
-        const s3Original = await client.upload({
-          Body: await image.toBuffer(),
-          Key: `project-background-images/${dirname}/original.${extension}`,
-          ContentType: inputs.file.type,
-        });
-        originalUrl = s3Original.Location;
-
-        const resizeBuffer = await image
-          .resize(
-            336,
-            200,
-            width < 336 || height < 200
-              ? {
-                  kernel: sharp.kernel.nearest,
-                }
-              : undefined,
-          )
-          .toBuffer();
-        const s3Thumb = await client.upload({
-          Body: resizeBuffer,
-          Key: `project-background-images/${dirname}/cover-336.${extension}`,
-          ContentType: inputs.file.type,
-        });
-        thumbUrl = s3Thumb.Location;
-      } catch (error1) {
-        try {
-          client.delete({ Key: `project-background-images/${dirname}/original.${extension}` });
-        } catch (error2) {
-          console.warn(error2.stack); // eslint-disable-line no-console
-        }
-
-        throw 'fileIsNotImage';
-      }
-
-      try {
-        rimraf.sync(inputs.file.fd);
-      } catch (error) {
-        console.warn(error.stack); // eslint-disable-line no-console
-      }
-
-      return {
-        dirname,
-        extension,
-        original: originalUrl,
-        thumb: thumbUrl,
-      };
-    }
-
-    const rootPath = path.join(sails.config.custom.projectBackgroundImagesPath, dirname);
-
-    fs.mkdirSync(rootPath);
-
     try {
-      await image.toFile(path.join(rootPath, `original.${extension}`));
+      const originalBuffer = await image.toBuffer();
 
-      await image
+      await fileManager.save(
+        `${folderPathSegment}/original.${extension}`,
+        originalBuffer,
+        inputs.file.type,
+      );
+
+      const cover336Buffer = await image
         .resize(
           336,
           200,
@@ -112,10 +61,18 @@ module.exports = {
               }
             : undefined,
         )
-        .toFile(path.join(rootPath, `cover-336.${extension}`));
+        .toBuffer();
+
+      await fileManager.save(
+        `${folderPathSegment}/cover-336.${extension}`,
+        cover336Buffer,
+        inputs.file.type,
+      );
     } catch (error1) {
+      console.warn(error1.stack); // eslint-disable-line no-console
+
       try {
-        rimraf.sync(rootPath);
+        fileManager.deleteFolder(folderPathSegment);
       } catch (error2) {
         console.warn(error2.stack); // eslint-disable-line no-console
       }
@@ -124,7 +81,7 @@ module.exports = {
     }
 
     try {
-      rimraf.sync(inputs.file.fd);
+      await rimraf(inputs.file.fd);
     } catch (error) {
       console.warn(error.stack); // eslint-disable-line no-console
     }

server/api/helpers/projects/update-one.js

@@ -1,6 +1,3 @@
-const path = require('path');
-const rimraf = require('rimraf');
-
 const valuesValidator = (value) => {
   if (!_.isPlainObject(value)) {
     return false;
@@ -86,27 +83,11 @@ module.exports = {
         (!project.backgroundImage ||
           project.backgroundImage.dirname !== inputs.record.backgroundImage.dirname)
       ) {
+        const fileManager = sails.hooks['file-manager'].getInstance();
+
         try {
-          if (sails.config.custom.s3Config) {
+          await fileManager.deleteFolder(
-            const client = await sails.helpers.utils.getSimpleStorageServiceClient();
+            `${sails.config.custom.projectBackgroundImagesPathSegment}/${inputs.record.backgroundImage.dirname}`,
-            if (client && inputs.record.backgroundImage && inputs.record.backgroundImage.original) {
-              const parsedUrl = new URL(inputs.record.backgroundImage.original);
-              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
-            }
-            if (client && inputs.record.backgroundImage && inputs.record.backgroundImage.thumb) {
-              const parsedUrl = new URL(inputs.record.backgroundImage.thumb);
-              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
-            }
-          }
-        } catch (error) {
-          console.warn(error.stack); // eslint-disable-line no-console
-        }
-        try {
-          rimraf.sync(
-            path.join(
-              sails.config.custom.projectBackgroundImagesPath,
-              inputs.record.backgroundImage.dirname,
-            ),
           );
         } catch (error) {
           console.warn(error.stack); // eslint-disable-line no-console

server/api/helpers/users/process-uploaded-avatar-file.js

@@ -1,6 +1,4 @@
-const fs = require('fs');
+const { rimraf } = require('rimraf');
-const path = require('path');
-const rimraf = require('rimraf');
 const { v4: uuid } = require('uuid');
 const sharp = require('sharp');
 
@@ -32,7 +30,10 @@ module.exports = {
       throw 'fileIsNotImage';
     }
 
+    const fileManager = sails.hooks['file-manager'].getInstance();
+
     const dirname = uuid();
+    const folderPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${dirname}`;
 
     let { width, pageHeight: height = metadata.height } = metadata;
     if (metadata.orientation && metadata.orientation > 4) {
@@ -41,68 +42,16 @@ module.exports = {
 
     const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
 
-    if (sails.config.custom.s3Config) {
-      const client = await sails.helpers.utils.getSimpleStorageServiceClient();
-      let originalUrl = '';
-      let squareUrl = '';
-
-      try {
-        const s3Original = await client.upload({
-          Body: await image.toBuffer(),
-          Key: `user-avatars/${dirname}/original.${extension}`,
-          ContentType: inputs.file.type,
-        });
-        originalUrl = s3Original.Location;
-
-        const resizeBuffer = await image
-          .resize(
-            100,
-            100,
-            width < 100 || height < 100
-              ? {
-                  kernel: sharp.kernel.nearest,
-                }
-              : undefined,
-          )
-          .toBuffer();
-        const s3Square = await client.upload({
-          Body: resizeBuffer,
-          Key: `user-avatars/${dirname}/square-100.${extension}`,
-          ContentType: inputs.file.type,
-        });
-        squareUrl = s3Square.Location;
-      } catch (error1) {
-        try {
-          client.delete({ Key: `user-avatars/${dirname}/original.${extension}` });
-        } catch (error2) {
-          console.warn(error2.stack); // eslint-disable-line no-console
-        }
-
-        throw 'fileIsNotImage';
-      }
-
-      try {
-        rimraf.sync(inputs.file.fd);
-      } catch (error) {
-        console.warn(error.stack); // eslint-disable-line no-console
-      }
-
-      return {
-        dirname,
-        extension,
-        original: originalUrl,
-        square: squareUrl,
-      };
-    }
-
-    const rootPath = path.join(sails.config.custom.userAvatarsPath, dirname);
-
-    fs.mkdirSync(rootPath);
-
     try {
-      await image.toFile(path.join(rootPath, `original.${extension}`));
+      const originalBuffer = await image.toBuffer();
 
-      await image
+      await fileManager.save(
+        `${folderPathSegment}/original.${extension}`,
+        originalBuffer,
+        inputs.file.type,
+      );
+
+      const square100Buffer = await image
         .resize(
           100,
           100,
@@ -112,10 +61,18 @@ module.exports = {
               }
             : undefined,
         )
-        .toFile(path.join(rootPath, `square-100.${extension}`));
+        .toBuffer();
+
+      await fileManager.save(
+        `${folderPathSegment}/square-100.${extension}`,
+        square100Buffer,
+        inputs.file.type,
+      );
     } catch (error1) {
+      console.warn(error1.stack); // eslint-disable-line no-console
+
       try {
-        rimraf.sync(rootPath);
+        fileManager.deleteFolder(folderPathSegment);
       } catch (error2) {
         console.warn(error2.stack); // eslint-disable-line no-console
       }
@@ -124,7 +81,7 @@ module.exports = {
     }
 
     try {
-      rimraf.sync(inputs.file.fd);
+      await rimraf(inputs.file.fd);
     } catch (error) {
       console.warn(error.stack); // eslint-disable-line no-console
     }

server/api/helpers/users/update-one.js

@@ -1,6 +1,4 @@
-const path = require('path');
 const bcrypt = require('bcrypt');
-const rimraf = require('rimraf');
 const { v4: uuid } = require('uuid');
 
 const valuesValidator = (value) => {
@@ -101,23 +99,12 @@ module.exports = {
         inputs.record.avatar &&
         (!user.avatar || user.avatar.dirname !== inputs.record.avatar.dirname)
       ) {
+        const fileManager = sails.hooks['file-manager'].getInstance();
+
         try {
-          if (sails.config.custom.s3Config) {
+          await fileManager.deleteFolder(
-            const client = await sails.helpers.utils.getSimpleStorageServiceClient();
+            `${sails.config.custom.userAvatarsPathSegment}/${inputs.record.avatar.dirname}`,
-            if (client && inputs.record.avatar && inputs.record.avatar.original) {
+          );
-              const parsedUrl = new URL(inputs.record.avatar.original);
-              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
-            }
-            if (client && inputs.record.avatar && inputs.record.avatar.square) {
-              const parsedUrl = new URL(inputs.record.avatar.square);
-              await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') });
-            }
-          }
-        } catch (error) {
-          console.warn(error.stack); // eslint-disable-line no-console
-        }
-        try {
-          rimraf.sync(path.join(sails.config.custom.userAvatarsPath, inputs.record.avatar.dirname));
         } catch (error) {
           console.warn(error.stack); // eslint-disable-line no-console
         }

server/api/helpers/utils/get-simple-storage-service-client.js

@@ -1,45 +0,0 @@
-const AWS = require('aws-sdk');
-
-class S3Client {
-  constructor(options) {
-    AWS.config.update({
-      accessKeyId: options.accessKeyId,
-      secretAccessKey: options.secretAccessKey,
-      region: options.region,
-    });
-    this.bucket = options.bucket;
-    this.client = new AWS.S3({
-      endpoint: options.endpoint,
-    });
-  }
-
-  upload({ Key, Body, ContentType }) {
-    return this.client
-      .upload({
-        Bucket: this.bucket,
-        Key,
-        Body,
-        ContentType,
-        ACL: 'public-read',
-      })
-      .promise();
-  }
-
-  delete({ Key }) {
-    return this.client
-      .deleteObject({
-        Bucket: this.bucket,
-        Key,
-      })
-      .promise();
-  }
-}
-
-module.exports = {
-  fn() {
-    if (sails.config.custom.s3Config) {
-      return new S3Client(sails.config.custom.s3Config);
-    }
-    return null;
-  },
-};

server/api/helpers/utils/receive-file.js

@@ -4,7 +4,7 @@ const { v4: uuid } = require('uuid');
 async function doUpload(paramName, req, options) {
   const uploadOptions = {
     ...options,
-    dirname: options.dirname || sails.config.custom.fileUploadTmpDir,
+    dirname: options.dirname || sails.config.custom.uploadsTempPath,
   };
   const upload = util.promisify((opts, callback) => {
     return req.file(paramName).upload(opts, (error, files) => callback(error, files));
@@ -33,7 +33,7 @@ module.exports = {
     exits.success(
       await doUpload(inputs.paramName, inputs.req, {
         saveAs: uuid(),
-        dirname: sails.config.custom.fileUploadTmpDir,
+        dirname: sails.config.custom.uploadsTempPath,
         maxBytes: null,
       }),
     );

server/api/hooks/file-manager/LocalFileManager.js

@@ -0,0 +1,51 @@
+const fs = require('fs');
+const fse = require('fs-extra');
+const path = require('path');
+const { rimraf } = require('rimraf');
+
+const PATH_SEGMENT_TO_URL_REPLACE_REGEX = /(public|private)\//;
+
+const buildPath = (pathSegment) => path.join(sails.config.custom.uploadsBasePath, pathSegment);
+
+class LocalFileManager {
+  // eslint-disable-next-line class-methods-use-this
+  async move(sourceFilePath, filePathSegment) {
+    const { dir, base } = path.parse(filePathSegment);
+
+    const folderPath = buildPath(dir);
+    const filePath = path.join(folderPath, base);
+
+    await fs.promises.mkdir(folderPath);
+    await fse.move(sourceFilePath, filePath);
+
+    return filePath;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  async save(filePathSegment, buffer) {
+    await fse.outputFile(buildPath(filePathSegment), buffer);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  read(filePathSegment) {
+    const filePath = buildPath(filePathSegment);
+
+    if (!fs.existsSync(filePath)) {
+      throw new Error('File does not exist');
+    }
+
+    return fs.createReadStream(filePath);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  async deleteFolder(folderPathSegment) {
+    await rimraf(buildPath(folderPathSegment));
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  buildUrl(filePathSegment) {
+    return `${sails.config.custom.baseUrl}/${filePathSegment.replace(PATH_SEGMENT_TO_URL_REPLACE_REGEX, '')}`;
+  }
+}
+
+module.exports = LocalFileManager;

server/api/hooks/file-manager/S3FileManager.js

@@ -0,0 +1,76 @@
+const fs = require('fs');
+const {
+  DeleteObjectsCommand,
+  GetObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+} = require('@aws-sdk/client-s3');
+
+class S3FileManager {
+  constructor(client) {
+    this.client = client;
+  }
+
+  async move(sourceFilePath, filePathSegment, contentType) {
+    const command = new PutObjectCommand({
+      Bucket: sails.config.custom.s3Bucket,
+      Key: filePathSegment,
+      Body: fs.createReadStream(sourceFilePath),
+      ContentType: contentType,
+    });
+
+    await this.client.send(command);
+
+    return null;
+  }
+
+  async save(filePathSegment, buffer, contentType) {
+    const command = new PutObjectCommand({
+      Bucket: sails.config.custom.s3Bucket,
+      Key: filePathSegment,
+      Body: buffer,
+      ContentType: contentType,
+    });
+
+    await this.client.send(command);
+  }
+
+  async read(filePathSegment) {
+    const command = new GetObjectCommand({
+      Bucket: sails.config.custom.s3Bucket,
+      Key: filePathSegment,
+    });
+
+    const result = await this.client.send(command);
+    return result.Body;
+  }
+
+  async deleteFolder(folderPathSegment) {
+    const listObjectsCommand = new ListObjectsV2Command({
+      Bucket: sails.config.custom.s3Bucket,
+      Prefix: folderPathSegment,
+    });
+
+    const result = await this.client.send(listObjectsCommand);
+
+    if (!result.Contents || result.Contents.length === 0) {
+      return;
+    }
+
+    const deleteObjectsCommand = new DeleteObjectsCommand({
+      Bucket: sails.config.custom.s3Bucket,
+      Delete: {
+        Objects: result.Contents.map(({ Key }) => ({ Key })),
+      },
+    });
+
+    await this.client.send(deleteObjectsCommand);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  buildUrl(filePathSegment) {
+    return `${sails.hooks.s3.getBaseUrl()}/${filePathSegment}`;
+  }
+}
+
+module.exports = S3FileManager;

server/api/hooks/file-manager/index.js

@@ -0,0 +1,41 @@
+const LocalFileManager = require('./LocalFileManager');
+const S3FileManager = require('./S3FileManager');
+
+/**
+ * file-manager hook
+ *
+ * @description :: A hook definition. Extends Sails by adding shadow routes, implicit actions,
+ *                 and/or initialization logic.
+ * @docs        :: https://sailsjs.com/docs/concepts/extending-sails/hooks
+ */
+
+module.exports = function defineFileManagerHook(sails) {
+  let instance = null;
+
+  const createInstance = () => {
+    instance = sails.hooks.s3.isActive()
+      ? new S3FileManager(sails.hooks.s3.getClient())
+      : new LocalFileManager();
+  };
+
+  return {
+    /**
+     * Runs when this Sails app loads/lifts.
+     */
+
+    async initialize() {
+      sails.log.info('Initializing custom hook (`file-manager`)');
+
+      return new Promise((resolve) => {
+        sails.after('hook:s3:loaded', () => {
+          createInstance();
+          resolve();
+        });
+      });
+    },
+
+    getInstance() {
+      return instance;
+    },
+  };
+};

server/api/hooks/s3/index.js

@@ -0,0 +1,64 @@
+const { URL } = require('url');
+const { S3Client } = require('@aws-sdk/client-s3');
+
+/**
+ * s3 hook
+ *
+ * @description :: A hook definition. Extends Sails by adding shadow routes, implicit actions,
+ *                 and/or initialization logic.
+ * @docs        :: https://sailsjs.com/docs/concepts/extending-sails/hooks
+ */
+
+module.exports = function defineS3Hook(sails) {
+  let client = null;
+
+  return {
+    /**
+     * Runs when this Sails app loads/lifts.
+     */
+
+    async initialize() {
+      if (!sails.config.custom.s3Endpoint && !sails.config.custom.s3Region) {
+        return;
+      }
+
+      sails.log.info('Initializing custom hook (`s3`)');
+
+      client = new S3Client({
+        endpoint: sails.config.custom.s3Endpoint,
+        region: sails.config.custom.s3Region || '-',
+        credentials: {
+          accessKeyId: sails.config.custom.s3AccessKeyId,
+          secretAccessKey: sails.config.custom.s3SecretAccessKey,
+        },
+        forcePathStyle: sails.config.custom.s3ForcePathStyle,
+      });
+    },
+
+    getClient() {
+      return client;
+    },
+
+    getBaseUrl() {
+      if (sails.config.custom.s3Endpoint) {
+        const { protocol, host } = new URL(sails.config.custom.s3Endpoint);
+
+        if (sails.config.custom.s3ForcePathStyle) {
+          return `${protocol}//${host}/${sails.config.custom.s3Bucket}`;
+        }
+
+        return `${protocol}//${sails.config.custom.s3Bucket}.${host}`;
+      }
+
+      if (sails.config.custom.s3ForcePathStyle) {
+        return `https://s3.${sails.config.custom.s3Region}.amazonaws.com/${sails.config.custom.s3Bucket}`;
+      }
+
+      return `https://${sails.config.custom.s3Bucket}.s3.${sails.config.custom.s3Region}.amazonaws.com`;
+    },
+
+    isActive() {
+      return client !== null;
+    },
+  };
+};

server/api/models/Attachment.js

@@ -26,17 +26,6 @@ module.exports = {
       type: 'string',
       required: true,
     },
-    type: {
-      type: 'string',
-    },
-    url: {
-      type: 'string',
-      allowNull: true,
-    },
-    thumb: {
-      type: 'string',
-      allowNull: true,
-    },
 
     //  ╔═╗╔╦╗╔╗ ╔═╗╔╦╗╔═╗
     //  ║╣ ║║║╠╩╗║╣  ║║╚═╗
@@ -59,20 +48,12 @@ module.exports = {
   },
 
   customToJSON() {
-    let { url, thumb } = this;
-    if (!url) {
-      url = `${sails.config.custom.attachmentsUrl}/${this.id}/download/${this.filename}`;
-    }
-    if (!thumb) {
-      thumb = this.image
-        ? `${sails.config.custom.attachmentsUrl}/${this.id}/download/thumbnails/cover-256.${this.image.thumbnailsExtension}`
-        : null;
-    }
-
     return {
-      ..._.omit(this, ['type', 'dirname', 'filename', 'image.thumbnailsExtension']),
+      ..._.omit(this, ['dirname', 'filename', 'image.thumbnailsExtension']),
-      url,
+      url: `${sails.config.custom.baseUrl}/attachments/${this.id}/download/${this.filename}`,
-      coverUrl: thumb,
+      coverUrl: this.image
+        ? `${sails.config.custom.baseUrl}/attachments/${this.id}/download/thumbnails/cover-256.${this.image.thumbnailsExtension}`
+        : null,
     };
   },
 };

server/api/models/Project.js

@@ -79,26 +79,13 @@ module.exports = {
   },
 
   customToJSON() {
-    let url = '';
+    const fileManager = sails.hooks['file-manager'].getInstance();
-    let coverUrl = '';
-    if (this.backgroundImage) {
-      if (this.backgroundImage.original) {
-        url = this.backgroundImage.original;
-      } else {
-        url = `${sails.config.custom.projectBackgroundImagesUrl}/${this.backgroundImage.dirname}/original.${this.backgroundImage.extension}`;
-      }
-      if (this.backgroundImage.thumb) {
-        coverUrl = this.backgroundImage.thumb;
-      } else {
-        coverUrl = `${sails.config.custom.projectBackgroundImagesUrl}/${this.backgroundImage.dirname}/cover-336.${this.backgroundImage.extension}`;
-      }
-    }
 
     return {
       ..._.omit(this, ['backgroundImage']),
       backgroundImage: this.backgroundImage && {
-        url,
+        url: `${fileManager.buildUrl(`${sails.config.custom.projectBackgroundImagesPathSegment}/${this.backgroundImage.dirname}/original.${this.backgroundImage.extension}`)}`,
-        coverUrl,
+        coverUrl: `${fileManager.buildUrl(`${sails.config.custom.projectBackgroundImagesPathSegment}/${this.backgroundImage.dirname}/cover-336.${this.backgroundImage.extension}`)}`,
       },
     };
   },

server/api/models/User.js

@@ -147,15 +147,8 @@ module.exports = {
   tableName: 'user_account',
 
   customToJSON() {
+    const fileManager = sails.hooks['file-manager'].getInstance();
     const isDefaultAdmin = this.email === sails.config.custom.defaultAdminEmail;
-    let avatarUrl = '';
-    if (this.avatar) {
-      if (this.avatar.square) {
-        avatarUrl = this.avatar.square;
-      } else {
-        avatarUrl = `${sails.config.custom.userAvatarsUrl}/${this.avatar.dirname}/square-100.${this.avatar.extension}`;
-      }
-    }
 
     return {
       ..._.omit(this, ['password', 'isSso', 'avatar', 'passwordChangedAt']),
@@ -163,7 +156,9 @@ module.exports = {
       isRoleLocked: (this.isSso && !sails.config.custom.oidcIgnoreRoles) || isDefaultAdmin,
       isUsernameLocked: (this.isSso && !sails.config.custom.oidcIgnoreUsername) || isDefaultAdmin,
       isDeletionLocked: isDefaultAdmin,
-      avatarUrl,
+      avatarUrl:
+        this.avatar &&
+        `${fileManager.buildUrl(`${sails.config.custom.userAvatarsPathSegment}/${this.avatar.dirname}/square-100.${this.avatar.extension}`)}`,
     };
   },
 };

server/config/custom.js

@@ -8,11 +8,10 @@
  * https://sailsjs.com/config/custom
  */
 
-const url = require('url');
+const { URL } = require('url');
-const path = require('path');
 const sails = require('sails');
 
-const parsedBasedUrl = new url.URL(process.env.BASE_URL);
+const parsedBasedUrl = new URL(process.env.BASE_URL);
 
 module.exports.custom = {
   /**
@@ -28,35 +27,26 @@ module.exports.custom = {
   tokenExpiresIn: parseInt(process.env.TOKEN_EXPIRES_IN, 10) || 365,
 
   // Location to receive uploaded files in. Default (non-string value) is a Sails-specific location.
-  fileUploadTmpDir: null,
+  uploadsTempPath: null,
+  uploadsBasePath: sails.config.appPath,
 
-  userAvatarsPath: path.join(sails.config.paths.public, 'user-avatars'),
+  userAvatarsPathSegment: 'public/user-avatars',
-  userAvatarsUrl: `${process.env.BASE_URL}/user-avatars`,
+  projectBackgroundImagesPathSegment: 'public/project-background-images',
-
+  attachmentsPathSegment: 'private/attachments',
-  projectBackgroundImagesPath: path.join(sails.config.paths.public, 'project-background-images'),
-  projectBackgroundImagesUrl: `${process.env.BASE_URL}/project-background-images`,
-
-  attachmentsPath: path.join(sails.config.appPath, 'private', 'attachments'),
-  attachmentsUrl: `${process.env.BASE_URL}/attachments`,
-
-  s3Config:
-    process.env.S3_ENABLE === 'true'
-      ? {
-          accessKeyId: process.env.S3_ACCESS_KEY,
-          secretAccessKey: process.env.S3_SECRET_KEY,
-          region: process.env.S3_REGION,
-          endpoint: process.env.S3_ENDPOINT,
-          bucket: process.env.S3_BUCKET,
-        }
-      : null,
 
   defaultAdminEmail:
     process.env.DEFAULT_ADMIN_EMAIL && process.env.DEFAULT_ADMIN_EMAIL.toLowerCase(),
 
   showDetailedAuthErrors: process.env.SHOW_DETAILED_AUTH_ERRORS === 'true',
-
   allowAllToCreateProjects: process.env.ALLOW_ALL_TO_CREATE_PROJECTS === 'true',
 
+  s3Endpoint: process.env.S3_ENDPOINT,
+  s3Region: process.env.S3_REGION,
+  s3AccessKeyId: process.env.S3_ACCESS_KEY_ID,
+  s3SecretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
+  s3Bucket: process.env.S3_BUCKET,
+  s3ForcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true',
+
   oidcIssuer: process.env.OIDC_ISSUER,
   oidcClientId: process.env.OIDC_CLIENT_ID,
   oidcClientSecret: process.env.OIDC_CLIENT_SECRET,

server/config/env/production.js

@@ -19,11 +19,11 @@
  * https://sailsjs.com/docs/concepts/deployment
  */
 
-const url = require('url');
+const { URL } = require('url');
 
 const { customLogger } = require('../../utils/logger');
 
-const parsedBasedUrl = new url.URL(process.env.BASE_URL);
+const parsedBasedUrl = new URL(process.env.BASE_URL);
 
 module.exports = {
   /**

server/config/routes.js

@@ -135,13 +135,21 @@ module.exports.routes = {
   'PATCH /api/notifications/:ids': 'notifications/update',
 
   'GET /user-avatars/*': {
-    fn: staticDirServer('/user-avatars', () => path.resolve(sails.config.custom.userAvatarsPath)),
+    fn: staticDirServer('/user-avatars', () =>
+      path.join(
+        path.resolve(sails.config.custom.uploadsBasePath),
+        sails.config.custom.userAvatarsPathSegment,
+      ),
+    ),
     skipAssets: false,
   },
 
   'GET /project-background-images/*': {
     fn: staticDirServer('/project-background-images', () =>
-      path.resolve(sails.config.custom.projectBackgroundImagesPath),
+      path.join(
+        path.resolve(sails.config.custom.uploadsBasePath),
+        sails.config.custom.projectBackgroundImagesPathSegment,
+      ),
     ),
     skipAssets: false,
   },

server/db/migrations/20241109164629_add_url_to_attachment_table.js

@@ -1,15 +0,0 @@
-module.exports.up = async (knex) => {
-  return knex.schema.table('attachment', (table) => {
-    table.text('type');
-    table.text('url');
-    table.text('thumb');
-  });
-};
-
-module.exports.down = async (knex) => {
-  return knex.schema.table('attachment', (table) => {
-    table.dropColumn('type');
-    table.dropColumn('url');
-    table.dropColumn('thumb');
-  });
-};

server/package.json

@@ -27,15 +27,15 @@
     }
   },
   "dependencies": {
-    "aws-sdk": "^2.1692.0",
+    "@aws-sdk/client-s3": "^3.688.0",
     "bcrypt": "^5.1.1",
     "dotenv": "^16.4.5",
     "dotenv-cli": "^7.4.2",
+    "fs-extra": "^11.2.0",
     "jsonwebtoken": "^9.0.2",
     "knex": "^3.1.0",
     "lodash": "^4.17.21",
     "moment": "^2.30.1",
-    "move-file": "^2.1.0",
     "nodemailer": "^6.9.15",
     "openid-client": "^5.7.0",
     "rimraf": "^5.0.10",