Transport controller and file model (#42)

* Transport controller and file model

* Use randomBytes intead of pseudoRandomBytes

* Cover all lines with tests

* Update code style

* Update code style

* View for image block

* Fix serving static files

* Mkdir -p for uploads dirs

* Add default secret param

* Add image Tool

* Update src/utils/objects.js

Co-Authored-By: talyguryn <vitalik7tv@yandex.ru>

* Use vars for image tool colors

* Revert var

* Remove --color-gray-border var

* Update src/controllers/transport.js

Co-Authored-By: talyguryn <vitalik7tv@yandex.ru>

* Add mp4 support for Image Tool

.gitignore

@@ -56,6 +56,7 @@ typings/
 
 # dotenv environment variables file
 .env
+.codexdocsrc
 
 # next.js build output
 .next
@@ -71,4 +72,7 @@ typings/
 .cache/
 .eslintcache
 .DS_Store
-.codexdocsrc
+
+# Uploads
+/public/uploads
+/public/uploads_test

config/development.json

@@ -1,4 +1,6 @@
 {
   "port": 3000,
-  "database": ".db"
+  "database": ".db",
+  "uploads": "public/uploads",
+  "secret": "iamasecretstring"
 }

config/index.js

@@ -16,6 +16,7 @@ if (fs.existsSync(path.resolve(__dirname, configPath))) {
   config = {
     database: '.db',
     port: 3000,
+    uploads: 'public/uploads',
     secret: 'secret'
   };
 }

config/production.json

@@ -1,4 +1,6 @@
 {
   "port": 3000,
-  "database": ".db"
+  "database": ".db",
+  "uploads": "public/uploads",
+  "secret": "iamasecretstring"
 }

config/testing.json

@@ -1,5 +1,7 @@
 {
   "port": 3001,
   "database": ".testdb",
-  "rcFile": "./test/.codexdocsrc"
+  "rcFile": "./test/.codexdocsrc",
+  "uploads": "public/uploads_test",
+  "secret": "iamasecretstring"
 }

package.json

@@ -23,14 +23,18 @@
     "csurf": "^1.9.0",
     "debug": "~4.1.0",
     "dotenv": "^6.2.0",
-    "jsonwebtoken": "^8.4.0",
     "eslint-plugin-standard": "^4.0.0",
     "express": "~4.16.0",
+    "file-type": "^10.7.1",
     "http-errors": "~1.7.1",
+    "jsonwebtoken": "^8.4.0",
+    "mime": "^2.4.0",
+    "mkdirp": "^0.5.1",
     "module-dispatcher": "^2.0.0",
     "morgan": "~1.9.0",
     "multer": "^1.3.1",
     "nedb": "^1.8.0",
+    "node-fetch": "^2.3.0",
     "nodemon": "^1.18.3",
     "normalize.css": "^8.0.0",
     "twig": "~1.12.0",
@@ -85,6 +89,7 @@
     "postcss-nested-ancestors": "^2.0.0",
     "postcss-nesting": "^7.0.0",
     "postcss-smart-import": "^0.7.6",
+    "rimraf": "^2.6.3",
     "sinon": "^7.0.0",
     "webpack": "^4.17.1",
     "webpack-cli": "^3.1.0"

public/dist/code-styling.css

@@ -1 +1 @@
-.hljs{display:block;overflow-x:auto;padding:.5em;color:#abb2bf;background:#282c34}.hljs-comment,.hljs-quote{color:#5c6370;font-style:italic}.hljs-doctag,.hljs-formula,.hljs-keyword{color:#c678dd}.hljs-deletion,.hljs-name,.hljs-section,.hljs-selector-tag,.hljs-subst{color:#e06c75}.hljs-literal{color:#56b6c2}.hljs-addition,.hljs-attribute,.hljs-meta-string,.hljs-regexp,.hljs-string{color:#98c379}.hljs-built_in,.hljs-class .hljs-title{color:#e6c07b}.hljs-attr,.hljs-number,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-pseudo,.hljs-template-variable,.hljs-type,.hljs-variable{color:#d19a66}.hljs-bullet,.hljs-link,.hljs-meta,.hljs-selector-id,.hljs-symbol,.hljs-title{color:#61aeee}.hljs-emphasis{font-style:italic}.hljs-strong{font-weight:700}.hljs-link{text-decoration:underline}
+.hljs{display:block;background:#fff;padding:.5em;color:#333;overflow-x:auto}.hljs-comment,.hljs-meta{color:#969896}.hljs-emphasis,.hljs-quote,.hljs-string,.hljs-strong,.hljs-template-variable,.hljs-variable{color:#df5000}.hljs-keyword,.hljs-selector-tag,.hljs-type{color:#a71d5d}.hljs-attribute,.hljs-bullet,.hljs-literal,.hljs-symbol{color:#0086b3}.hljs-name,.hljs-section{color:#63a35c}.hljs-tag{color:#333}.hljs-attr,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-id,.hljs-selector-pseudo,.hljs-title{color:#795da3}.hljs-addition{color:#55a532;background-color:#eaffea}.hljs-deletion{color:#bd2c00;background-color:#ffecec}.hljs-link{text-decoration:underline}

src/app.js

@@ -4,7 +4,6 @@ const path = require('path');
 const cookieParser = require('cookie-parser');
 const logger = require('morgan');
 const rcParser = require('./utils/rcparser');
-
 const routes = require('./routes');
 
 const app = express();

src/controllers/transport.js

@@ -0,0 +1,119 @@
+const fileType = require('file-type');
+const fetch = require('node-fetch');
+const fs = require('fs');
+const nodePath = require('path');
+
+const Model = require('../models/file');
+const { random16 } = require('../utils/crypto');
+const { deepMerge } = require('../utils/objects');
+const config = require('../../config');
+
+/**
+ * @class Transport
+ * @classdesc Transport controller
+ *
+ * Allows to save files from client or fetch them by URL
+ */
+class Transport {
+  /**
+   * Saves file passed from client
+   * @param {object} multerData - file data from multer
+   * @param {string} multerData.originalname - original name of the file
+   * @param {string} multerData.filename - name of the uploaded file
+   * @param {string} multerData.path - path to the uploaded file
+   * @param {number} multerData.size - size of the uploaded file
+   * @param {string} multerData.mimetype - MIME type of the uploaded file
+   *
+   * @param {object} map - object that represents how should fields of File object should be mapped to response
+   * @return {Promise<FileData>}
+   */
+  static async save(multerData, map) {
+    const { originalname: name, path, filename, size, mimetype } = multerData;
+
+    const file = new Model({ name, filename, path, size, mimetype });
+
+    await file.save();
+
+    let response = file.data;
+
+    if (map) {
+      response = Transport.composeResponse(file, map);
+    }
+
+    return response;
+  }
+
+  /**
+   * Fetches file by passed URL
+   * @param {string} url - URL of the file
+   * @param {object} map - object that represents how should fields of File object should be mapped to response
+   * @return {Promise<FileData>}
+   */
+  static async fetch(url, map) {
+    const fetchedFile = await fetch(url);
+    const buffer = await fetchedFile.buffer();
+    const filename = await random16();
+
+    const type = fileType(buffer);
+    const ext = type ? type.ext : nodePath.extname(url).slice(1);
+
+    fs.writeFileSync(`${config.uploads}/${filename}.${ext}`, buffer);
+
+    const file = new Model({
+      name: url,
+      filename: `${filename}.${ext}`,
+      path: `${config.uploads}/${filename}.${ext}`,
+      size: buffer.length,
+      mimetype: type ? type.mime : fetchedFile.headers.get('content-type')
+    });
+
+    await file.save();
+
+    let response = file.data;
+
+    if (map) {
+      response = Transport.composeResponse(file, map);
+    }
+
+    return response;
+  }
+
+  /**
+   * Map fields of File object to response by provided map object
+   *
+   * @param {File} file
+   * @param {object} map - object that represents how should fields of File object should be mapped to response
+   *
+   */
+  static composeResponse(file, map) {
+    const response = {};
+    const { data } = file;
+
+    Object.entries(map).forEach(([name, path]) => {
+      const fields = path.split(':');
+
+      if (fields.length > 1) {
+        let object = {};
+        let result = object;
+
+        fields.forEach((field, i) => {
+          if (i === fields.length - 1) {
+            object[field] = data[name];
+            return;
+          }
+
+          object[field] = {};
+          object = object[field];
+        });
+
+        deepMerge(response, result);
+      } else {
+        response[fields[0]] = data[name];
+      }
+    });
+
+    return response;
+  }
+}
+
+module.exports = Transport;

src/frontend/js/classes/codeStyler.js

@@ -2,8 +2,7 @@ import hljs from 'highlight.js/lib/highlight';
 import javascript from 'highlight.js/lib/languages/javascript';
 import xml from 'highlight.js/lib/languages/xml';
 import json from 'highlight.js/lib/languages/json';
-// eslint-disable-next-line
+import style from 'highlight.js/styles/github-gist.css'; // eslint-disable-line no-unused-vars
-import style from 'highlight.js/styles/atom-one-dark.css';
 
 /**
  * @class CodeStyles

src/frontend/js/classes/editor.js

@@ -10,6 +10,7 @@ import CodeTool from '@editorjs/code';
 import Delimiter from '@editorjs/delimiter';
 import InlineCode from '@editorjs/inline-code';
 import List from '@editorjs/list';
+import Image from '@editorjs/image';
 import RawTool from '@editorjs/raw';
 import Embed from '@editorjs/embed';
 
@@ -34,20 +35,6 @@ export default class Editor {
             placeholder: options.headerPlaceholder || ''
           }
         },
-        // image: {
-        //   class: ImageTool,
-        //   inlineToolbar: true,
-        //   config: {
-        //     endpoints: {
-        //       byFile: '/editor/transport',
-        //       byUrl: '/editor/transport'
-        //     }
-        //   }
-        // },
-        list: {
-          class: List,
-          inlineToolbar: true
-        },
         quote: {
           class: Quote,
           inlineToolbar: true
@@ -69,6 +56,28 @@ export default class Editor {
         marker: {
           class: Marker,
           shortcut: 'CMD+SHIFT+M'
+        },
+        list: {
+          class: List,
+          inlineToolbar: true
+        },
+        image: {
+          class: Image,
+          inlineToolbar: true,
+          config: {
+            types: 'image/*, video/mp4',
+            endpoints: {
+              byFile: '/api/transport/image',
+              byUrl: '/api/transport/fetch'
+            },
+            additionalRequestData: {
+              map: JSON.stringify({
+                path: 'file:url',
+                size: 'file:size',
+                mimetype: 'file:mime'
+              })
+            }
+          }
         }
       },
       data: {

src/frontend/styles/components/page.pcss

@@ -116,6 +116,54 @@
   }
 }
 
+/**
+ * Image
+ * ==================
+ */
+.block-image {
+ margin: 40px auto;
+ text-align: center;
+
+ &__content {
+   img {
+     vertical-align: bottom;
+     max-width: 100%;
+   }
+
+   &--stretched {
+     max-width: none !important;
+     width: calc(100% + 120px) !important;
+     margin-left: -60px;
+
+     img {
+       width: 100%;
+     }
+   }
+
+   &--bordered {
+     img {
+       border: 3px solid var(--color-line-gray);
+       box-sizing: border-box;
+     }
+   }
+
+   &--with-background {
+     padding: 15px;
+     background: var(--color-line-gray);
+
+     img {
+       max-width: 60%;
+       margin: 0 auto;
+     }
+   }
+ }
+
+ &__caption {
+   margin: 1em auto;
+   color: var(--color-text-second);
+ }
+}
+
 /**
  * Delimiter
  * ==================

src/frontend/styles/vars.pcss

@@ -4,7 +4,6 @@
   --color-line-gray: #E8E8EB;
   --color-link-active: #388AE5;
   --color-button-danger: #ff1629;
-  --color-gray-border: rgba(var(--color-line-gray), 0.48);
 
   /**
    * Site layout sizes

src/models/alias.js

@@ -1,5 +1,5 @@
 const { aliases: aliasesDb } = require('../utils/database/index');
-const binaryMD5 = require('../utils/crypto');
+const { binaryMD5 } = require('../utils/crypto');
 
 /**
  * @typedef {Object} AliasData

src/models/file.js

@@ -0,0 +1,157 @@
+const { files: filesDb } = require('../utils/database/index');
+
+/**
+ * @typedef {Object} FileData
+ *
+ * @property {string} _id - file id
+ * @property {string} name - original file name
+ * @property {string} filename - name of uploaded file
+ * @property {string} path - path to uploaded file
+ * @property {string} mimetype - file MIME type
+ * @property {number} size - size of the file in
+ */
+
+/**
+ * @class File
+ * @class File model
+ *
+ * @property {string} _id - file id
+ * @property {string} name - original file name
+ * @property {string} filename - name of uploaded file
+ * @property {string} path - path to uploaded file
+ * @property {string} mimetype - file MIME type
+ * @property {number} size - size of the file in
+ */
+class File {
+  /**
+   * Find and return model of file with given id
+   * @param {string} _id - file id
+   * @returns {Promise<File>}
+   */
+  static async get(_id) {
+    const data = await filesDb.findOne({ _id });
+
+    return new File(data);
+  }
+
+  /**
+   * Find and return model of file with given id
+   * @param {string} filename - uploaded filename
+   * @returns {Promise<File>}
+   */
+  static async getByFilename(filename) {
+    const data = await filesDb.findOne({ filename });
+
+    return new File(data);
+  }
+
+  /**
+   * Find all files which match passed query object
+   *
+   * @param {Object} query
+   * @returns {Promise<File[]>}
+   */
+  static async getAll(query = {}) {
+    const docs = await filesDb.find(query);
+
+    return Promise.all(docs.map(doc => new File(doc)));
+  }
+
+  /**
+   * @constructor
+   *
+   * @param {FileData} data
+   */
+  constructor(data = {}) {
+    if (data === null) {
+      data = {};
+    }
+
+    if (data._id) {
+      this._id = data._id;
+    }
+
+    this.data = data;
+  }
+
+  /**
+   * Set FileData object fields to internal model fields
+   *
+   * @param {FileData} fileData
+   */
+  set data(fileData) {
+    const { name, filename, path, mimetype, size } = fileData;
+
+    this.name = name || this.name;
+    this.filename = filename || this.filename;
+    this.path = path ? this.processPath(path) : this.path;
+    this.mimetype = mimetype || this.mimetype;
+    this.size = size || this.size;
+  }
+
+  /**
+   * Return FileData object
+   *
+   * @returns {FileData}
+   */
+  get data() {
+    return {
+      _id: this._id,
+      name: this.name,
+      filename: this.filename,
+      path: this.path,
+      mimetype: this.mimetype,
+      size: this.size
+    };
+  }
+
+  /**
+   * Save or update file data in the database
+   *
+   * @returns {Promise<File>}
+   */
+  async save() {
+    if (!this._id) {
+      const insertedRow = await filesDb.insert(this.data);
+
+      this._id = insertedRow._id;
+    } else {
+      await filesDb.update({ _id: this._id }, this.data);
+    }
+
+    return this;
+  }
+
+  /**
+   * Remove file data from the database
+   *
+   * @returns {Promise<File>}
+   */
+  async destroy() {
+    await filesDb.remove({ _id: this._id });
+
+    delete this._id;
+
+    return this;
+  }
+
+  /**
+   * Removes unnecessary public folder prefix
+   * @param {string} path
+   * @return {string}
+   */
+  processPath(path) {
+    return path.replace(/^public/, '');
+  }
+
+  /**
+   * Return readable file data
+   *
+   * @returns {FileData}
+   */
+  toJSON() {
+    return this.data;
+  }
+}
+
+module.exports = File;

src/routes/api/index.js

@@ -2,7 +2,9 @@ const express = require('express');
 const router = express.Router();
 
 const pagesAPI = require('./pages');
+const transportAPI = require('./transport');
 
 router.use('/', pagesAPI);
+router.use('/', transportAPI);
 
 module.exports = router;

src/routes/api/pages.js

@@ -3,6 +3,7 @@ const router = express.Router();
 const multer = require('multer')();
 const Pages = require('../../controllers/pages');
 const PagesOrder = require('../../controllers/pagesOrder');
+
 /**
  * GET /page/:id
  *
@@ -51,7 +52,7 @@ router.get('/pages', async (req, res) => {
  *
  * Create new page in the database
  */
-router.put('/page', multer.any(), async (req, res) => {
+router.put('/page', multer.none(), async (req, res) => {
   try {
     const { title, body, parent } = req.body;
     const page = await Pages.insert({ title, body, parent });
@@ -76,7 +77,7 @@ router.put('/page', multer.any(), async (req, res) => {
  *
  * Update page data in the database
  */
-router.post('/page/:id', multer.any(), async (req, res) => {
+router.post('/page/:id', multer.none(), async (req, res) => {
   const { id } = req.params;
 
   try {
@@ -136,7 +137,7 @@ router.delete('/page/:id', async (req, res) => {
      * @param startFrom
      * @returns {Promise<void>}
      */
-    const deleteRecursively = async function (startFrom) {
+    const deleteRecursively = async (startFrom) => {
       let order = [];
 
       try {

src/routes/api/transport.js

@@ -0,0 +1,119 @@
+const express = require('express');
+const router = express.Router();
+const multer = require('multer');
+const mime = require('mime');
+const mkdirp = require('mkdirp');
+const Transport = require('../../controllers/transport');
+const { random16 } = require('../../utils/crypto');
+const config = require('../../../config');
+
+/**
+ * Multer storage for uploaded files and images
+ * @type {DiskStorage|DiskStorage}
+ */
+const storage = multer.diskStorage({
+  destination: (req, file, cb) => {
+    const dir = config.uploads || 'public/uploads';
+
+    mkdirp(dir, err => cb(err, dir));
+  },
+  filename: async (req, file, cb) => {
+    const filename = await random16();
+
+    cb(null, `${filename}.${mime.getExtension(file.mimetype)}`);
+  }
+});
+
+/**
+ * Multer middleware for image uploading
+ */
+const imageUploader = multer({
+  storage,
+  fileFilter: (req, file, cb) => {
+    if (!/image/.test(file.mimetype) && !/video\/mp4/.test(file.mimetype)) {
+      cb(null, false);
+      return;
+    }
+
+    cb(null, true);
+  }
+}).fields([ { name: 'image', maxCount: 1 } ]);
+
+/**
+ * Multer middleware for file uploading
+ */
+const fileUploader = multer({
+  storage
+}).fields([ { name: 'file', maxCount: 1 } ]);
+
+/**
+ * Accepts images to upload
+ */
+router.post('/transport/image', imageUploader, async (req, res) => {
+  let response = { success: 0 };
+
+  if (!req.files || !req.files.image) {
+    res.status(400).json(response);
+    return;
+  }
+
+  try {
+    Object.assign(
+      response,
+      await Transport.save(req.files.image[0], req.body.map ? JSON.parse(req.body.map) : undefined)
+    );
+
+    response.success = 1;
+    res.status(200).json(response);
+  } catch (e) {
+    res.status(500).json(response);
+  }
+});
+
+/**
+ * Accepts files to upload
+ */
+router.post('/transport/file', fileUploader, async (req, res) => {
+  let response = { success: 0 };
+
+  if (!req.files || !req.files.file) {
+    res.status(400).json(response);
+    return;
+  }
+
+  try {
+    Object.assign(
+      response,
+      await Transport.save(req.files.file[0], req.body.map ? JSON.parse(req.body.map) : undefined)
+    );
+
+    response.success = 1;
+    res.status(200).json(response);
+  } catch (e) {
+    res.status(500).json(response);
+  }
+});
+
+/**
+ * Accept file url to fetch
+ */
+router.post('/transport/fetch', multer().none(), async (req, res) => {
+  let response = { success: 0 };
+
+  if (!req.body.url) {
+    res.status(400).json(response);
+    return;
+  }
+
+  try {
+    Object.assign(response, await Transport.fetch(req.body.url, req.body.map ? JSON.parse(req.body.map) : undefined));
+
+    response.success = 1;
+    res.status(200).json(response);
+  } catch (e) {
+    console.log(e);
+    res.status(500).json(response);
+  }
+});
+
+module.exports = router;

src/utils/crypto.js

@@ -5,8 +5,29 @@ const crypto = require('crypto');
  * @param stringToHash - string to hash
  * @returns {string} - binary hash of argument
  */
-module.exports = function binaryMD5(stringToHash) {
+function binaryMD5(stringToHash) {
   return crypto.createHash('md5')
     .update(stringToHash)
     .digest('binary');
+}
+
+/**
+ * Returns 16 random bytes in hex format
+ * @return {Promise<string>}
+ */
+function random16() {
+  return new Promise((resolve, reject) => {
+    crypto.randomBytes(16, (err, raw) => {
+      if (err) {
+        reject(err);
+      }
+
+      resolve(raw.toString('hex'));
+    });
+  });
+}
+
+module.exports = {
+  binaryMD5,
+  random16
 };

src/utils/database/files.js

@@ -0,0 +1,6 @@
+const Datastore = require('nedb');
+const config = require('../../../config');
+
+const db = new Datastore({ filename: `./${config.database}/files.db`, autoload: true });
+
+module.exports = db;

src/utils/database/index.js

@@ -1,4 +1,5 @@
 const pages = require('./pages');
+const files = require('./files');
 const password = require('./password');
 const aliases = require('./aliases');
 const pagesOrder = require('./pagesOrder');
@@ -148,5 +149,6 @@ module.exports = {
   pages: new Database(pages),
   password: new Database(password),
   aliases: new Database(aliases),
-  pagesOrder: new Database(pagesOrder)
+  pagesOrder: new Database(pagesOrder),
+  files: new Database(files)
 };

src/utils/objects.js

@@ -0,0 +1,32 @@
+/**
+ * Merge to objects recursively
+ * @param {object} target
+ * @param {object[]} sources
+ * @return {object}
+ */
+function deepMerge(target, ...sources) {
+  const isObject = item => item && typeof item === 'object' && !Array.isArray(item);
+
+  if (!sources.length) return target;
+  const source = sources.shift();
+
+  if (isObject(target) && isObject(source)) {
+    for (const key in source) {
+      if (isObject(source[key])) {
+        if (!target[key]) {
+          Object.assign(target, { [key]: {} });
+        }
+
+        deepMerge(target[key], source[key]);
+      } else {
+        Object.assign(target, { [key]: source[key] });
+      }
+    }
+  }
+
+  return deepMerge(target, ...sources);
+}
+
+module.exports = {
+  deepMerge
+};

src/views/pages/blocks/image.twig

@@ -0,0 +1,24 @@
+{% set classes = ['block-image__content'] %}
+
+{% if withBorder %}
+  {% set classes = classes|merge(['block-image__content--bordered']) %}
+{% endif %}
+
+{% if stretched %}
+  {% set classes = classes|merge(['block-image__content--stretched']) %}
+{% endif %}
+
+{% if withBackground %}
+  {% set classes = classes|merge(['block-image__content--with-background']) %}
+{% endif %}
+
+<figure class="block-image">
+  <div class="{{classes.join(' ')}}">
+    <img src="{{file.url}}" alt="{{caption ? caption : ''}}">
+  </div>
+  {% if caption %}
+      <footer class="block-image__caption">
+          {{caption}}
+      </footer>
+  {% endif %}
+</figure>

src/views/pages/page.twig

@@ -32,7 +32,7 @@
       {% for block in page.body.blocks %}
         {# Skip first header, because it is already showed as a Title #}
         {% if not (loop.first and block.type == 'header') %}
-          {% if block.type in ['paragraph', 'header', 'list', 'code', 'delimiter'] %}
+          {% if block.type in ['paragraph', 'header', 'list', 'code', 'image', 'delimiter'] %}
             {% include './blocks/' ~ block.type ~ '.twig' with block.data %}
           {% endif %}
         {% endif %}

test/models/alias.js

@@ -3,7 +3,7 @@ const fs = require('fs');
 const path = require('path');
 const config = require('../../config');
 const Alias = require('../../src/models/alias');
-const binaryMD5 = require('../../src/utils/crypto');
+const {binaryMD5} = require('../../src/utils/crypto');
 const {aliases} = require('../../src/utils/database');
 
 describe('Alias model', () => {

test/models/file.js

@@ -0,0 +1,233 @@
+const {expect} = require('chai');
+const fs = require('fs');
+const path = require('path');
+const config = require('../../config');
+const File = require('../../src/models/file');
+const {files} = require('../../src/utils/database');
+
+describe('File model', () => {
+
+  after(() => {
+    const pathToDB = path.resolve(__dirname, '../../', config.database, './files.db');
+
+    if (fs.existsSync(pathToDB)) {
+      fs.unlinkSync(pathToDB);
+    }
+  });
+
+  it('Working with empty model', async () => {
+    let file = new File();
+
+    expect(file.data).to.be.a('object');
+
+    let {data} = file;
+
+    expect(data._id).to.be.undefined;
+    expect(data.name).to.be.undefined;
+    expect(data.filename).to.be.undefined;
+    expect(data.path).to.be.undefined;
+    expect(data.size).to.be.undefined;
+    expect(data.mimetype).to.be.undefined;
+
+    file = new File(null);
+
+    data = file.data;
+
+    expect(data._id).to.be.undefined;
+    expect(data.name).to.be.undefined;
+    expect(data.filename).to.be.undefined;
+    expect(data.path).to.be.undefined;
+    expect(data.size).to.be.undefined;
+    expect(data.mimetype).to.be.undefined;
+
+    const initialData = {
+      _id: 'file_id',
+      name: 'filename',
+      filename: 'randomname',
+      path: '/uploads/randomname',
+      size: 1024,
+      mimetype: 'image/png'
+    };
+
+    file = new File(initialData);
+
+    const json = file.toJSON();
+
+    data = file.data;
+
+    expect(data._id).to.equal(initialData._id);
+    expect(data.name).to.equal(initialData.name);
+    expect(data.filename).to.equal(initialData.filename);
+    expect(data.path).to.equal(initialData.path);
+    expect(data.size).to.equal(initialData.size);
+    expect(data.mimetype).to.equal(initialData.mimetype);
+
+    const update = {
+      _id: 12345,
+      name: 'updated filename',
+      filename: 'updated randomname',
+      path: '/uploads/updated randomname',
+      size: 2048,
+      mimetype: 'image/jpeg'
+    };
+
+    file.data = update;
+
+    data = file.data;
+
+    expect(data._id).to.equal(initialData._id);
+    expect(data.name).to.equal(update.name);
+    expect(data.filename).to.equal(update.filename);
+    expect(data.path).to.equal(update.path);
+    expect(data.size).to.equal(update.size);
+    expect(data.mimetype).to.equal(update.mimetype);
+  });
+
+  it('Saving, updating and deleting model in the database', async () => {
+    const initialData = {
+      name: 'filename',
+      filename: 'randomname',
+      path: '/uploads/randomname',
+      size: 1024,
+      mimetype: 'image/png'
+    };
+
+    const file = new File(initialData);
+
+    let savedFile = await file.save();
+
+    expect(savedFile._id).not.be.undefined;
+    expect(savedFile.name).to.equal(initialData.name);
+    expect(savedFile.filename).to.equal(initialData.filename);
+    expect(savedFile.path).to.equal(initialData.path);
+    expect(savedFile.size).to.equal(initialData.size);
+    expect(savedFile.mimetype).to.equal(initialData.mimetype);
+
+    const insertedFile = await files.findOne({_id: file._id});
+
+    expect(insertedFile._id).to.equal(file._id);
+    expect(insertedFile.name).to.equal(file.name);
+    expect(insertedFile.filename).to.equal(file.filename);
+    expect(insertedFile.path).to.equal(file.path);
+    expect(insertedFile.size).to.equal(file.size);
+    expect(insertedFile.mimetype).to.equal(file.mimetype);
+
+    const updateData = {
+      _id: 12345,
+      name: 'updated filename',
+      filename: 'updated randomname',
+      path: '/uploads/updated randomname',
+      size: 2048,
+      mimetype: 'image/jpeg'
+    };
+
+    file.data = updateData;
+    await file.save();
+
+    expect(file._id).to.equal(insertedFile._id);
+
+    const updatedFile = await files.findOne({_id: file._id});
+
+    expect(updatedFile._id).to.equal(savedFile._id);
+    expect(updatedFile.name).to.equal(updateData.name);
+    expect(updatedFile.filename).to.equal(updateData.filename);
+    expect(updatedFile.path).to.equal(updateData.path);
+    expect(updatedFile.size).to.equal(updateData.size);
+    expect(updatedFile.mimetype).to.equal(updateData.mimetype);
+
+    await file.destroy();
+
+    expect(file._id).to.be.undefined;
+
+    const removedFile = await files.findOne({_id: updatedFile._id});
+
+    expect(removedFile).to.be.null;
+  });
+
+  it('Static get method', async () => {
+    const initialData = {
+      name: 'filename',
+      filename: 'randomname',
+      path: '/uploads/randomname',
+      size: 1024,
+      mimetype: 'image/png'
+    };
+
+    const file = new File(initialData);
+
+    const savedFile = await file.save();
+
+    const foundFile = await File.get(savedFile._id);
+
+    const {data} = foundFile;
+
+    expect(data._id).to.equal(savedFile._id);
+    expect(data.name).to.equal(savedFile.name);
+    expect(data.filename).to.equal(savedFile.filename);
+    expect(data.path).to.equal(savedFile.path);
+    expect(data.size).to.equal(savedFile.size);
+    expect(data.mimetype).to.equal(savedFile.mimetype);
+
+    await file.destroy();
+  });
+
+  it('Static getByFilename method', async () => {
+    const initialData = {
+      name: 'filename',
+      filename: 'randomname',
+      path: '/uploads/randomname',
+      size: 1024,
+      mimetype: 'image/png'
+    };
+
+    const file = new File(initialData);
+
+    const savedFile = await file.save();
+
+    const foundFile = await File.getByFilename(savedFile.filename);
+
+    const {data} = foundFile;
+
+    expect(data._id).to.equal(savedFile._id);
+    expect(data.name).to.equal(savedFile.name);
+    expect(data.filename).to.equal(savedFile.filename);
+    expect(data.path).to.equal(savedFile.path);
+    expect(data.size).to.equal(savedFile.size);
+    expect(data.mimetype).to.equal(savedFile.mimetype);
+
+    await file.destroy();
+  });
+
+  it('Static getAll method', async () => {
+    const filesToSave = [
+      new File({
+        name: 'filename1',
+        filename: 'randomname1',
+        path: '/uploads/randomname1',
+        size: 1024,
+        mimetype: 'image/png'
+      }),
+      new File({
+        name: 'filename2',
+        filename: 'randomname2',
+        path: '/uploads/randomname2',
+        size: 2048,
+        mimetype: 'image/jpeg'
+      }),
+    ];
+
+    const savedFiles = await Promise.all(filesToSave.map(file => file.save()));
+
+    const foundFiles = await File.getAll({_id: {$in: savedFiles.map(file => file._id)}});
+
+    expect(foundFiles.length).to.equal(2);
+
+    foundFiles.forEach((file, i) => {
+      expect(file.name).to.equal(filesToSave[i].name);
+      expect(file.filename).to.equal(filesToSave[i].filename);
+      expect(file.path).to.equal(filesToSave[i].path);
+      expect(file.size).to.equal(filesToSave[i].size);
+      expect(file.mimetype).to.equal(filesToSave[i].mimetype);
+    });
+  });
+});

test/rest/test_file.json

@@ -0,0 +1,3 @@
+{
+  "Hello": "world"
+}

test/rest/transport.js

@@ -0,0 +1,259 @@
+const fs = require('fs');
+const path = require('path');
+const fileType = require('file-type');
+const chai = require('chai');
+const chaiHTTP = require('chai-http');
+const rimraf = require('rimraf');
+const {expect} = chai;
+
+const {app} = require('../../bin/www');
+const model = require('../../src/models/file');
+
+const config = require('../../config');
+
+chai.use(chaiHTTP);
+
+describe('Transport routes: ', () => {
+  let agent;
+
+  before(async () => {
+    agent = chai.request.agent(app);
+
+    if (!fs.existsSync('./' + config.uploads)) {
+      fs.mkdirSync('./' + config.uploads);
+    }
+  });
+
+  after(async () => {
+    const pathToDB = path.resolve(__dirname, '../../', config.database, './files.db');
+
+    if (fs.existsSync(pathToDB)) {
+      fs.unlinkSync(pathToDB);
+    }
+
+    if (fs.existsSync('./' + config.uploads)) {
+      rimraf.sync('./' + config.uploads);
+    }
+  });
+
+  it('Uploading an image', async () => {
+    const name = 'test_image.png';
+    const image = fs.readFileSync(path.resolve(`./test/rest/${name}`));
+    const res = await agent
+      .post('/api/transport/image')
+      .attach('image', image, name);
+
+    expect(res).to.have.status(200);
+    expect(res).to.be.json;
+
+    const { body } = res;
+
+    const file = await model.get(body._id);
+
+    expect(body.success).to.equal(1);
+    expect(file._id).to.equal(body._id);
+    expect(file.name).to.equal(name);
+    expect(file.filename).to.equal(body.filename);
+    expect(file.path).to.equal(body.path);
+    expect(file.mimetype).to.equal(fileType(image).mime);
+    expect(file.size).to.equal(image.byteLength);
+
+    const getRes = await agent
+      .get(file.path);
+
+    expect(getRes).to.have.status(200);
+    expect(getRes).to.have.header('content-type', fileType(image).mime);
+  });
+
+  it('Uploading an image with map option', async () => {
+    const name = 'test_image.png';
+    const image = fs.readFileSync(path.resolve(`./test/rest/${name}`));
+    const res = await agent
+      .post('/api/transport/image')
+      .attach('image', image, name)
+      .field('map', JSON.stringify({_id: '_id', path: 'file:url', size: 'file:size', name: 'file:name'}));
+
+    expect(res).to.have.status(200);
+    expect(res).to.be.json;
+
+    const { body } = res;
+
+    const file = await model.get(body._id);
+
+    expect(body.success).to.equal(1);
+    expect(file.name).to.equal(body.file.name);
+    expect(file.path).to.equal(body.file.url);
+    expect(file.size).to.equal(body.file.size);
+  });
+
+  it('Uploading a file', async () => {
+    const name = 'test_file.json';
+    const json = fs.readFileSync(path.resolve(`./test/rest/${name}`));
+    const res = await agent
+      .post('/api/transport/file')
+      .attach('file', json, name);
+
+    expect(res).to.have.status(200);
+    expect(res).to.be.json;
+
+    const { body } = res;
+
+    const file = await model.get(body._id);
+
+    expect(body.success).to.equal(1);
+    expect(file._id).to.equal(body._id);
+    expect(file.name).to.equal(name);
+    expect(file.filename).to.equal(body.filename);
+    expect(file.path).to.equal(body.path);
+    expect(file.size).to.equal(json.byteLength);
+
+    const getRes = await agent
+      .get(file.path);
+
+    expect(getRes).to.have.status(200);
+    expect(getRes).to.have.header('content-type', new RegExp(`^${file.mimetype}`));
+  });
+
+  it('Uploading a file with map option', async () => {
+    const name = 'test_file.json';
+    const json = fs.readFileSync(path.resolve(`./test/rest/${name}`));
+    const res = await agent
+      .post('/api/transport/file')
+      .attach('file', json, name)
+      .field('map', JSON.stringify({_id: '_id', path: 'file:url', size: 'file:size', name: 'file:name'}));
+
+    expect(res).to.have.status(200);
+    expect(res).to.be.json;
+
+    const { body } = res;
+
+    const file = await model.get(body._id);
+
+    expect(body.success).to.equal(1);
+    expect(file.name).to.equal(body.file.name);
+    expect(file.path).to.equal(body.file.url);
+    expect(file.size).to.equal(body.file.size);
+  });
+
+  it('Send file URL to fetch', async () => {
+    const url = 'https://codex.so/public/app/img/codex-logo.svg';
+    const res = await agent
+      .post('/api/transport/fetch')
+      .field('url', url);
+
+    expect(res).to.have.status(200);
+    expect(res).to.be.json;
+
+    const { body } = res;
+
+    const file = await model.get(body._id);
+
+    expect(body.success).to.equal(1);
+    expect(file._id).to.equal(body._id);
+    expect(file.name).to.equal(body.name);
+    expect(file.filename).to.equal(body.filename);
+    expect(file.path).to.equal(body.path);
+    expect(file.size).to.equal(body.size);
+
+    const getRes = await agent
+      .get(file.path);
+
+    expect(getRes).to.have.status(200);
+    expect(getRes).to.have.header('content-type', file.mimetype);
+  });
+
+  it('Send an file URL to fetch with map option', async () => {
+    const url = 'https://codex.so/public/app/img/codex-logo.svg';
+    const res = await agent
+      .post('/api/transport/fetch')
+      .field('url', url)
+      .field('map', JSON.stringify({_id: '_id', path: 'file:url', size: 'file:size', name: 'file:name'}));
+
+    expect(res).to.have.status(200);
+    expect(res).to.be.json;
+
+    const { body } = res;
+
+    const file = await model.get(body._id);
+
+    expect(body.success).to.equal(1);
+    expect(file.name).to.equal(body.file.name);
+    expect(file.path).to.equal(body.file.url);
+    expect(file.size).to.equal(body.file.size);
+  });
+
+  it('Negative tests for file uploading', async () => {
+    let res = await agent
+      .post('/api/transport/file')
+      .send();
+
+    let {body} = res;
+
+    expect(res).to.have.status(400);
+    expect(body.success).to.equal(0);
+
+    const name = 'test_file.json';
+    const json = fs.readFileSync(path.resolve(`./test/rest/${name}`));
+    res = await agent
+      .post('/api/transport/file')
+      .attach('file', json, name)
+      .field('map', '{unvalid_json)');
+
+    body = res.body;
+
+    expect(res).to.have.status(500);
+    expect(body.success).to.equal(0);
+  });
+
+  it('Negative tests for image uploading', async () => {
+    let res = await agent
+      .post('/api/transport/image')
+      .send();
+
+    let {body} = res;
+
+    expect(res).to.have.status(400);
+    expect(body.success).to.equal(0);
+
+    let name = 'test_file.json';
+    const json = fs.readFileSync(path.resolve(`./test/rest/${name}`));
+    res = await agent
+      .post('/api/transport/image')
+      .attach('image', json, name);
+
+    expect(res).to.have.status(400);
+
+    name = 'test_image.png';
+    const image = fs.readFileSync(path.resolve(`./test/rest/${name}`));
+    res = await agent
+      .post('/api/transport/image')
+      .attach('image', image, name)
+      .field('map', '{unvalid_json)');
+
+    body = res.body;
+
+    expect(res).to.have.status(500);
+    expect(body.success).to.equal(0);
+  });
+
+  it('Negative tests for file fetching', async () => {
+    let res = await agent
+      .post('/api/transport/fetch')
+      .send();
+
+    let {body} = res;
+
+    expect(res).to.have.status(400);
+    expect(body.success).to.equal(0);
+
+    const url = 'https://invalidurl';
+    res = await agent
+      .post('/api/transport/fetch')
+      .field('url', url);
+
+    body = res.body;
+
+    expect(res).to.have.status(500);
+    expect(body.success).to.equal(0);
+  }).timeout(50000);
+});

yarn.lock

@@ -2906,6 +2906,11 @@ file-entry-cache@^2.0.0:
     flat-cache "^1.2.1"
     object-assign "^4.0.1"
 
+file-type@^10.7.1:
+  version "10.7.1"
+  resolved "https://registry.yarnpkg.com/file-type/-/file-type-10.7.1.tgz#bcfdd618fddfa7f7e5fc504e08b62cfec7bda8f2"
+  integrity sha512-kUc4EE9q3MH6kx70KumPOvXLZLEJZzY9phEVg/bKWyGZ+OA9KoKZzFR4HS0yDmNv31sJkdf4hbTERIfplF9OxQ==
+
 fill-range@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7"
@@ -4438,6 +4443,11 @@ mime@^1.4.1:
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
   integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
 
+mime@^2.4.0:
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.0.tgz#e051fd881358585f3279df333fe694da0bcffdd6"
+  integrity sha512-ikBcWwyqXQSHKtciCcctu9YfPbFYZ4+gbHEmE0Q8jzcTYQg5dHCr3g2wwAZjPoJfQVXZq6KXAjpXOTf5/cjT7w==
+
 mimic-fn@^1.0.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022"
@@ -4687,6 +4697,11 @@ nise@^1.4.7:
     path-to-regexp "^1.7.0"
     text-encoding "^0.6.4"
 
+node-fetch@^2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.3.0.tgz#1a1d940bbfb916a1d3e0219f037e89e71f8c5fa5"
+  integrity sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA==
+
 node-libs-browser@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/node-libs-browser/-/node-libs-browser-2.1.0.tgz#5f94263d404f6e44767d726901fff05478d600df"
@@ -6254,7 +6269,7 @@ rgba-regex@^1.0.0:
   resolved "https://registry.yarnpkg.com/rgba-regex/-/rgba-regex-1.0.0.tgz#43374e2e2ca0968b0ef1523460b7d730ff22eeb3"
   integrity sha1-QzdOLiyglosO8VI0YLfXMP8i7rM=
 
-rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@~2.6.2:
+rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3, rimraf@~2.6.2:
   version "2.6.3"
   resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab"
   integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==