Transport controller and file model (#42)

* Transport controller and file model

* Use randomBytes intead of pseudoRandomBytes

* Cover all lines with tests

* Update code style

* Update code style

* View for image block

* Fix serving static files

* Mkdir -p for uploads dirs

* Add default secret param

* Add image Tool

* Update src/utils/objects.js

Co-Authored-By: talyguryn <vitalik7tv@yandex.ru>

* Use vars for image tool colors

* Revert var

* Remove --color-gray-border var

* Update src/controllers/transport.js

Co-Authored-By: talyguryn <vitalik7tv@yandex.ru>

* Add mp4 support for Image Tool

src/app.js

@@ -4,7 +4,6 @@ const path = require('path');
 const cookieParser = require('cookie-parser');
 const logger = require('morgan');
 const rcParser = require('./utils/rcparser');
-
 const routes = require('./routes');
 
 const app = express();

src/controllers/transport.js

@@ -0,0 +1,119 @@
+const fileType = require('file-type');
+const fetch = require('node-fetch');
+const fs = require('fs');
+const nodePath = require('path');
+
+const Model = require('../models/file');
+const { random16 } = require('../utils/crypto');
+const { deepMerge } = require('../utils/objects');
+const config = require('../../config');
+
+/**
+ * @class Transport
+ * @classdesc Transport controller
+ *
+ * Allows to save files from client or fetch them by URL
+ */
+class Transport {
+  /**
+   * Saves file passed from client
+   * @param {object} multerData - file data from multer
+   * @param {string} multerData.originalname - original name of the file
+   * @param {string} multerData.filename - name of the uploaded file
+   * @param {string} multerData.path - path to the uploaded file
+   * @param {number} multerData.size - size of the uploaded file
+   * @param {string} multerData.mimetype - MIME type of the uploaded file
+   *
+   * @param {object} map - object that represents how should fields of File object should be mapped to response
+   * @return {Promise<FileData>}
+   */
+  static async save(multerData, map) {
+    const { originalname: name, path, filename, size, mimetype } = multerData;
+
+    const file = new Model({ name, filename, path, size, mimetype });
+
+    await file.save();
+
+    let response = file.data;
+
+    if (map) {
+      response = Transport.composeResponse(file, map);
+    }
+
+    return response;
+  }
+
+  /**
+   * Fetches file by passed URL
+   * @param {string} url - URL of the file
+   * @param {object} map - object that represents how should fields of File object should be mapped to response
+   * @return {Promise<FileData>}
+   */
+  static async fetch(url, map) {
+    const fetchedFile = await fetch(url);
+    const buffer = await fetchedFile.buffer();
+    const filename = await random16();
+
+    const type = fileType(buffer);
+    const ext = type ? type.ext : nodePath.extname(url).slice(1);
+
+    fs.writeFileSync(`${config.uploads}/${filename}.${ext}`, buffer);
+
+    const file = new Model({
+      name: url,
+      filename: `${filename}.${ext}`,
+      path: `${config.uploads}/${filename}.${ext}`,
+      size: buffer.length,
+      mimetype: type ? type.mime : fetchedFile.headers.get('content-type')
+    });
+
+    await file.save();
+
+    let response = file.data;
+
+    if (map) {
+      response = Transport.composeResponse(file, map);
+    }
+
+    return response;
+  }
+
+  /**
+   * Map fields of File object to response by provided map object
+   *
+   * @param {File} file
+   * @param {object} map - object that represents how should fields of File object should be mapped to response
+   *
+   */
+  static composeResponse(file, map) {
+    const response = {};
+    const { data } = file;
+
+    Object.entries(map).forEach(([name, path]) => {
+      const fields = path.split(':');
+
+      if (fields.length > 1) {
+        let object = {};
+        let result = object;
+
+        fields.forEach((field, i) => {
+          if (i === fields.length - 1) {
+            object[field] = data[name];
+            return;
+          }
+
+          object[field] = {};
+          object = object[field];
+        });
+
+        deepMerge(response, result);
+      } else {
+        response[fields[0]] = data[name];
+      }
+    });
+
+    return response;
+  }
+}
+
+module.exports = Transport;

src/frontend/js/classes/codeStyler.js

@@ -2,8 +2,7 @@ import hljs from 'highlight.js/lib/highlight';
 import javascript from 'highlight.js/lib/languages/javascript';
 import xml from 'highlight.js/lib/languages/xml';
 import json from 'highlight.js/lib/languages/json';
-// eslint-disable-next-line
-import style from 'highlight.js/styles/atom-one-dark.css';
+import style from 'highlight.js/styles/github-gist.css'; // eslint-disable-line no-unused-vars
 
 /**
  * @class CodeStyles

src/frontend/js/classes/editor.js

@@ -10,6 +10,7 @@ import CodeTool from '@editorjs/code';
 import Delimiter from '@editorjs/delimiter';
 import InlineCode from '@editorjs/inline-code';
 import List from '@editorjs/list';
+import Image from '@editorjs/image';
 import RawTool from '@editorjs/raw';
 import Embed from '@editorjs/embed';
 
@@ -34,20 +35,6 @@ export default class Editor {
             placeholder: options.headerPlaceholder || ''
           }
         },
-        // image: {
-        //   class: ImageTool,
-        //   inlineToolbar: true,
-        //   config: {
-        //     endpoints: {
-        //       byFile: '/editor/transport',
-        //       byUrl: '/editor/transport'
-        //     }
-        //   }
-        // },
-        list: {
-          class: List,
-          inlineToolbar: true
-        },
         quote: {
           class: Quote,
           inlineToolbar: true
@@ -69,6 +56,28 @@ export default class Editor {
         marker: {
           class: Marker,
           shortcut: 'CMD+SHIFT+M'
+        },
+        list: {
+          class: List,
+          inlineToolbar: true
+        },
+        image: {
+          class: Image,
+          inlineToolbar: true,
+          config: {
+            types: 'image/*, video/mp4',
+            endpoints: {
+              byFile: '/api/transport/image',
+              byUrl: '/api/transport/fetch'
+            },
+            additionalRequestData: {
+              map: JSON.stringify({
+                path: 'file:url',
+                size: 'file:size',
+                mimetype: 'file:mime'
+              })
+            }
+          }
         }
       },
       data: {

src/frontend/styles/components/page.pcss

@@ -116,6 +116,54 @@
   }
 }
 
+/**
+ * Image
+ * ==================
+ */
+.block-image {
+ margin: 40px auto;
+ text-align: center;
+
+ &__content {
+   img {
+     vertical-align: bottom;
+     max-width: 100%;
+   }
+
+   &--stretched {
+     max-width: none !important;
+     width: calc(100% + 120px) !important;
+     margin-left: -60px;
+
+     img {
+       width: 100%;
+     }
+   }
+
+   &--bordered {
+     img {
+       border: 3px solid var(--color-line-gray);
+       box-sizing: border-box;
+     }
+   }
+
+   &--with-background {
+     padding: 15px;
+     background: var(--color-line-gray);
+
+     img {
+       max-width: 60%;
+       margin: 0 auto;
+     }
+   }
+ }
+
+ &__caption {
+   margin: 1em auto;
+   color: var(--color-text-second);
+ }
+}
+
 /**
  * Delimiter
  * ==================

src/frontend/styles/vars.pcss

@@ -4,7 +4,6 @@
   --color-line-gray: #E8E8EB;
   --color-link-active: #388AE5;
   --color-button-danger: #ff1629;
-  --color-gray-border: rgba(var(--color-line-gray), 0.48);
 
   /**
    * Site layout sizes

src/models/alias.js

@@ -1,5 +1,5 @@
 const { aliases: aliasesDb } = require('../utils/database/index');
-const binaryMD5 = require('../utils/crypto');
+const { binaryMD5 } = require('../utils/crypto');
 
 /**
  * @typedef {Object} AliasData

src/models/file.js

@@ -0,0 +1,157 @@
+const { files: filesDb } = require('../utils/database/index');
+
+/**
+ * @typedef {Object} FileData
+ *
+ * @property {string} _id - file id
+ * @property {string} name - original file name
+ * @property {string} filename - name of uploaded file
+ * @property {string} path - path to uploaded file
+ * @property {string} mimetype - file MIME type
+ * @property {number} size - size of the file in
+ */
+
+/**
+ * @class File
+ * @class File model
+ *
+ * @property {string} _id - file id
+ * @property {string} name - original file name
+ * @property {string} filename - name of uploaded file
+ * @property {string} path - path to uploaded file
+ * @property {string} mimetype - file MIME type
+ * @property {number} size - size of the file in
+ */
+class File {
+  /**
+   * Find and return model of file with given id
+   * @param {string} _id - file id
+   * @returns {Promise<File>}
+   */
+  static async get(_id) {
+    const data = await filesDb.findOne({ _id });
+
+    return new File(data);
+  }
+
+  /**
+   * Find and return model of file with given id
+   * @param {string} filename - uploaded filename
+   * @returns {Promise<File>}
+   */
+  static async getByFilename(filename) {
+    const data = await filesDb.findOne({ filename });
+
+    return new File(data);
+  }
+
+  /**
+   * Find all files which match passed query object
+   *
+   * @param {Object} query
+   * @returns {Promise<File[]>}
+   */
+  static async getAll(query = {}) {
+    const docs = await filesDb.find(query);
+
+    return Promise.all(docs.map(doc => new File(doc)));
+  }
+
+  /**
+   * @constructor
+   *
+   * @param {FileData} data
+   */
+  constructor(data = {}) {
+    if (data === null) {
+      data = {};
+    }
+
+    if (data._id) {
+      this._id = data._id;
+    }
+
+    this.data = data;
+  }
+
+  /**
+   * Set FileData object fields to internal model fields
+   *
+   * @param {FileData} fileData
+   */
+  set data(fileData) {
+    const { name, filename, path, mimetype, size } = fileData;
+
+    this.name = name || this.name;
+    this.filename = filename || this.filename;
+    this.path = path ? this.processPath(path) : this.path;
+    this.mimetype = mimetype || this.mimetype;
+    this.size = size || this.size;
+  }
+
+  /**
+   * Return FileData object
+   *
+   * @returns {FileData}
+   */
+  get data() {
+    return {
+      _id: this._id,
+      name: this.name,
+      filename: this.filename,
+      path: this.path,
+      mimetype: this.mimetype,
+      size: this.size
+    };
+  }
+
+  /**
+   * Save or update file data in the database
+   *
+   * @returns {Promise<File>}
+   */
+  async save() {
+    if (!this._id) {
+      const insertedRow = await filesDb.insert(this.data);
+
+      this._id = insertedRow._id;
+    } else {
+      await filesDb.update({ _id: this._id }, this.data);
+    }
+
+    return this;
+  }
+
+  /**
+   * Remove file data from the database
+   *
+   * @returns {Promise<File>}
+   */
+  async destroy() {
+    await filesDb.remove({ _id: this._id });
+
+    delete this._id;
+
+    return this;
+  }
+
+  /**
+   * Removes unnecessary public folder prefix
+   * @param {string} path
+   * @return {string}
+   */
+  processPath(path) {
+    return path.replace(/^public/, '');
+  }
+
+  /**
+   * Return readable file data
+   *
+   * @returns {FileData}
+   */
+  toJSON() {
+    return this.data;
+  }
+}
+
+module.exports = File;

src/routes/api/index.js

@@ -2,7 +2,9 @@ const express = require('express');
 const router = express.Router();
 
 const pagesAPI = require('./pages');
+const transportAPI = require('./transport');
 
 router.use('/', pagesAPI);
+router.use('/', transportAPI);
 
 module.exports = router;

src/routes/api/pages.js

@@ -3,6 +3,7 @@ const router = express.Router();
 const multer = require('multer')();
 const Pages = require('../../controllers/pages');
 const PagesOrder = require('../../controllers/pagesOrder');
+
 /**
  * GET /page/:id
  *
@@ -51,7 +52,7 @@ router.get('/pages', async (req, res) => {
  *
  * Create new page in the database
  */
-router.put('/page', multer.any(), async (req, res) => {
+router.put('/page', multer.none(), async (req, res) => {
   try {
     const { title, body, parent } = req.body;
     const page = await Pages.insert({ title, body, parent });
@@ -76,7 +77,7 @@ router.put('/page', multer.any(), async (req, res) => {
  *
  * Update page data in the database
  */
-router.post('/page/:id', multer.any(), async (req, res) => {
+router.post('/page/:id', multer.none(), async (req, res) => {
   const { id } = req.params;
 
   try {
@@ -136,7 +137,7 @@ router.delete('/page/:id', async (req, res) => {
      * @param startFrom
      * @returns {Promise<void>}
      */
-    const deleteRecursively = async function (startFrom) {
+    const deleteRecursively = async (startFrom) => {
       let order = [];
 
       try {

src/routes/api/transport.js

@@ -0,0 +1,119 @@
+const express = require('express');
+const router = express.Router();
+const multer = require('multer');
+const mime = require('mime');
+const mkdirp = require('mkdirp');
+const Transport = require('../../controllers/transport');
+const { random16 } = require('../../utils/crypto');
+const config = require('../../../config');
+
+/**
+ * Multer storage for uploaded files and images
+ * @type {DiskStorage|DiskStorage}
+ */
+const storage = multer.diskStorage({
+  destination: (req, file, cb) => {
+    const dir = config.uploads || 'public/uploads';
+
+    mkdirp(dir, err => cb(err, dir));
+  },
+  filename: async (req, file, cb) => {
+    const filename = await random16();
+
+    cb(null, `${filename}.${mime.getExtension(file.mimetype)}`);
+  }
+});
+
+/**
+ * Multer middleware for image uploading
+ */
+const imageUploader = multer({
+  storage,
+  fileFilter: (req, file, cb) => {
+    if (!/image/.test(file.mimetype) && !/video\/mp4/.test(file.mimetype)) {
+      cb(null, false);
+      return;
+    }
+
+    cb(null, true);
+  }
+}).fields([ { name: 'image', maxCount: 1 } ]);
+
+/**
+ * Multer middleware for file uploading
+ */
+const fileUploader = multer({
+  storage
+}).fields([ { name: 'file', maxCount: 1 } ]);
+
+/**
+ * Accepts images to upload
+ */
+router.post('/transport/image', imageUploader, async (req, res) => {
+  let response = { success: 0 };
+
+  if (!req.files || !req.files.image) {
+    res.status(400).json(response);
+    return;
+  }
+
+  try {
+    Object.assign(
+      response,
+      await Transport.save(req.files.image[0], req.body.map ? JSON.parse(req.body.map) : undefined)
+    );
+
+    response.success = 1;
+    res.status(200).json(response);
+  } catch (e) {
+    res.status(500).json(response);
+  }
+});
+
+/**
+ * Accepts files to upload
+ */
+router.post('/transport/file', fileUploader, async (req, res) => {
+  let response = { success: 0 };
+
+  if (!req.files || !req.files.file) {
+    res.status(400).json(response);
+    return;
+  }
+
+  try {
+    Object.assign(
+      response,
+      await Transport.save(req.files.file[0], req.body.map ? JSON.parse(req.body.map) : undefined)
+    );
+
+    response.success = 1;
+    res.status(200).json(response);
+  } catch (e) {
+    res.status(500).json(response);
+  }
+});
+
+/**
+ * Accept file url to fetch
+ */
+router.post('/transport/fetch', multer().none(), async (req, res) => {
+  let response = { success: 0 };
+
+  if (!req.body.url) {
+    res.status(400).json(response);
+    return;
+  }
+
+  try {
+    Object.assign(response, await Transport.fetch(req.body.url, req.body.map ? JSON.parse(req.body.map) : undefined));
+
+    response.success = 1;
+    res.status(200).json(response);
+  } catch (e) {
+    console.log(e);
+    res.status(500).json(response);
+  }
+});
+
+module.exports = router;

src/utils/crypto.js

@@ -5,8 +5,29 @@ const crypto = require('crypto');
  * @param stringToHash - string to hash
  * @returns {string} - binary hash of argument
  */
-module.exports = function binaryMD5(stringToHash) {
+function binaryMD5(stringToHash) {
   return crypto.createHash('md5')
     .update(stringToHash)
     .digest('binary');
+}
+
+/**
+ * Returns 16 random bytes in hex format
+ * @return {Promise<string>}
+ */
+function random16() {
+  return new Promise((resolve, reject) => {
+    crypto.randomBytes(16, (err, raw) => {
+      if (err) {
+        reject(err);
+      }
+
+      resolve(raw.toString('hex'));
+    });
+  });
+}
+
+module.exports = {
+  binaryMD5,
+  random16
 };

src/utils/database/files.js

@@ -0,0 +1,6 @@
+const Datastore = require('nedb');
+const config = require('../../../config');
+
+const db = new Datastore({ filename: `./${config.database}/files.db`, autoload: true });
+
+module.exports = db;

src/utils/database/index.js

@@ -1,4 +1,5 @@
 const pages = require('./pages');
+const files = require('./files');
 const password = require('./password');
 const aliases = require('./aliases');
 const pagesOrder = require('./pagesOrder');
@@ -148,5 +149,6 @@ module.exports = {
   pages: new Database(pages),
   password: new Database(password),
   aliases: new Database(aliases),
-  pagesOrder: new Database(pagesOrder)
+  pagesOrder: new Database(pagesOrder),
+  files: new Database(files)
 };

src/utils/objects.js

@@ -0,0 +1,32 @@
+/**
+ * Merge to objects recursively
+ * @param {object} target
+ * @param {object[]} sources
+ * @return {object}
+ */
+function deepMerge(target, ...sources) {
+  const isObject = item => item && typeof item === 'object' && !Array.isArray(item);
+
+  if (!sources.length) return target;
+  const source = sources.shift();
+
+  if (isObject(target) && isObject(source)) {
+    for (const key in source) {
+      if (isObject(source[key])) {
+        if (!target[key]) {
+          Object.assign(target, { [key]: {} });
+        }
+
+        deepMerge(target[key], source[key]);
+      } else {
+        Object.assign(target, { [key]: source[key] });
+      }
+    }
+  }
+
+  return deepMerge(target, ...sources);
+}
+
+module.exports = {
+  deepMerge
+};

src/views/pages/blocks/image.twig

@@ -0,0 +1,24 @@
+{% set classes = ['block-image__content'] %}
+
+{% if withBorder %}
+  {% set classes = classes|merge(['block-image__content--bordered']) %}
+{% endif %}
+
+{% if stretched %}
+  {% set classes = classes|merge(['block-image__content--stretched']) %}
+{% endif %}
+
+{% if withBackground %}
+  {% set classes = classes|merge(['block-image__content--with-background']) %}
+{% endif %}
+
+<figure class="block-image">
+  <div class="{{classes.join(' ')}}">
+    <img src="{{file.url}}" alt="{{caption ? caption : ''}}">
+  </div>
+  {% if caption %}
+      <footer class="block-image__caption">
+          {{caption}}
+      </footer>
+  {% endif %}
+</figure>

src/views/pages/page.twig

@@ -32,7 +32,7 @@
       {% for block in page.body.blocks %}
         {# Skip first header, because it is already showed as a Title #}
         {% if not (loop.first and block.type == 'header') %}
-          {% if block.type in ['paragraph', 'header', 'list', 'code', 'delimiter'] %}
+          {% if block.type in ['paragraph', 'header', 'list', 'code', 'image', 'delimiter'] %}
             {% include './blocks/' ~ block.type ~ '.twig' with block.data %}
           {% endif %}
         {% endif %}