🤩MongoDB support 🤩 (#272)

* implement configuration through YAML

* remove rcparser

* use password from appConfig

* update docker configs

* fix dockerignore

* implement mongodb driver

* update eslint packages

* fix bugs

* refactor code for grouping by parent

* fix yet another bug

* use unique symbol to the EntityId type

* fix more bugs

* implement db converter

* fix bug with parent selector

* fix eslint

* db-converter refactoring

* create cli program for db-converter

* add readme and gitignore

* update development docs

* update development docs and default config

* add docs about converter

* add src/test to docker ignore

* move database code from utils

* improve docs

* eslint fix

* add more docs

* fix docs

* remove env_file from docker-compose

* implement duplicate detection in db-converter

* use published version of the config-loader

* fix bug

* Update DEVELOPMENT.md

Co-authored-by: Ilya Maroz <37909603+ilyamore88@users.noreply.github.com>

* fix bugs

* fix next/prev buttons

* fix more bugs

* fix sorting

Co-authored-by: Ilya Maroz <37909603+ilyamore88@users.noreply.github.com>

src/backend/database/index.ts

@@ -0,0 +1,50 @@
+import { AliasData } from '../models/alias.js';
+import { FileData } from '../models/file.js';
+import { PageData } from '../models/page.js';
+import { PageOrderData } from '../models/pageOrder.js';
+import appConfig from '../utils/appConfig.js';
+import LocalDatabaseDriver from './local.js';
+import MongoDatabaseDriver from './mongodb.js';
+import { EntityId } from './types.js';
+import { ObjectId } from 'mongodb';
+
+const Database = appConfig.database.driver === 'mongodb' ? MongoDatabaseDriver : LocalDatabaseDriver;
+
+/**
+ * Convert a string to an EntityId (string or ObjectId depending on the database driver)
+ *
+ * @param id - id to convert
+ */
+export function toEntityId(id: string): EntityId {
+  if (id === '0') {
+    return id as EntityId;
+  }
+
+  return (appConfig.database.driver === 'mongodb' ? new ObjectId(id) : id) as EntityId;
+}
+
+/**
+ * Check if provided ids are equal
+ *
+ * @param id1 - first id
+ * @param id2 - second id
+ */
+export function isEqualIds(id1?: EntityId, id2?: EntityId): boolean {
+  return id1?.toString() === id2?.toString();
+}
+
+/**
+ * Check if provided ids are valid
+ *
+ * @param id - id to check
+ */
+export function isEntityId(id?: EntityId): id is EntityId {
+  return typeof id === 'string' || id instanceof ObjectId;
+}
+
+export default {
+  pages: new Database<PageData>('pages'),
+  aliases: new Database<AliasData>('aliases'),
+  pagesOrder: new Database<PageOrderData>('pagesOrder'),
+  files: new Database<FileData>('files'),
+};

src/backend/database/local.ts

@@ -0,0 +1,173 @@
+import Datastore from 'nedb';
+import { DatabaseDriver, Options } from './types.js';
+import path from 'path';
+import appConfig from '../utils/appConfig.js';
+
+/**
+ * Init function for nedb instance
+ *
+ * @param {string} name - name of the data file
+ * @returns {Datastore} db - nedb instance
+ */
+function initDb(name: string): Datastore {
+  const dbConfig = appConfig.database.driver === 'local' ? appConfig.database.local : null;
+
+  if (!dbConfig) {
+    throw new Error('Database config is not initialized');
+  }
+
+  return new Datastore({
+    filename: path.resolve(`${dbConfig.path}/${name}.db`),
+    autoload: true,
+  });
+}
+
+/**
+ * Resolve function helper
+ */
+export interface ResolveFunction {
+  (value: any): void;
+}
+
+/**
+ * Reject function helper
+ */
+export interface RejectFunction {
+  (reason?: unknown): void;
+}
+
+
+/**
+ * Simple decorator class to work with nedb datastore
+ */
+export default class LocalDatabaseDriver<DocType> implements DatabaseDriver<DocType> {
+  /**
+   * nedb Datastore object
+   */
+  private db: Datastore;
+
+  /**
+   * @param collectionName - collection name for storing data
+   */
+  constructor(collectionName: string) {
+    this.db = initDb(collectionName);
+  }
+
+  /**
+   * Insert new document into the database
+   *
+   * @see https://github.com/louischatriot/nedb#inserting-documents
+   * @param {object} doc - object to insert
+   * @returns {Promise<object | Error>} - inserted doc or Error object
+   */
+  public async insert(doc: DocType): Promise<DocType> {
+    return new Promise((resolve, reject) => this.db.insert(doc, (err, newDoc) => {
+      if (err) {
+        reject(err);
+      }
+
+      resolve(newDoc);
+    }));
+  }
+
+  /**
+   * Find documents that match passed query
+   *
+   * @see https://github.com/louischatriot/nedb#finding-documents
+   * @param {object} query - query object
+   * @param {object} projection - projection object
+   * @returns {Promise<Array<object> | Error>} - found docs or Error object
+   */
+  public async find(query: Record<string, unknown>, projection?: DocType): Promise<Array<DocType>> {
+    const cbk = (resolve: ResolveFunction, reject: RejectFunction) => (err: Error | null, docs: DocType[]) => {
+      if (err) {
+        reject(err);
+      }
+
+      resolve(docs);
+    };
+
+    return new Promise((resolve, reject) => {
+      if (projection) {
+        this.db.find(query, projection, cbk(resolve, reject));
+      } else {
+        this.db.find(query, cbk(resolve, reject));
+      }
+    });
+  }
+
+  /**
+   * Find one document matches passed query
+   *
+   * @see https://github.com/louischatriot/nedb#finding-documents
+   * @param {object} query - query object
+   * @param {object} projection - projection object
+   * @returns {Promise<object | Error>} - found doc or Error object
+   */
+  public async findOne(query: Record<string, unknown>, projection?: DocType): Promise<DocType> {
+    const cbk = (resolve: ResolveFunction, reject: RejectFunction) => (err: Error | null, doc: DocType) => {
+      if (err) {
+        reject(err);
+      }
+
+      resolve(doc);
+    };
+
+    return new Promise((resolve, reject) => {
+      if (projection) {
+        this.db.findOne(query, projection, cbk(resolve, reject));
+      } else {
+        this.db.findOne(query, cbk(resolve, reject));
+      }
+    });
+  }
+
+  /**
+   * Update document matches query
+   *
+   * @see https://github.com/louischatriot/nedb#updating-documents
+   * @param {object} query - query object
+   * @param {object} update - fields to update
+   * @param {Options} options - optional params
+   * @returns {Promise<number | object | object[] | Error>} - number of updated rows or affected docs or Error object
+   */
+  public async update(query: Record<string, unknown>, update: DocType, options: Options = {}): Promise<number|boolean|Array<DocType>> {
+    return new Promise((resolve, reject) => this.db.update(query, update, options, (err, result, affectedDocs) => {
+      if (err) {
+        reject(err);
+      }
+
+      switch (true) {
+        case options.returnUpdatedDocs:
+          resolve(affectedDocs);
+          break;
+        case options.upsert:
+          if (affectedDocs) {
+            resolve(affectedDocs);
+          }
+          resolve(result);
+          break;
+        default:
+          resolve(result);
+      }
+    }));
+  }
+
+  /**
+   * Remove document matches passed query
+   *
+   * @see https://github.com/louischatriot/nedb#removing-documents
+   * @param {object} query - query object
+   * @param {Options} options - optional params
+   * @returns {Promise<number|Error>} - number of removed rows or Error object
+   */
+  public async remove(query: Record<string, unknown>, options: Options = {}): Promise<number> {
+    return new Promise((resolve, reject) => this.db.remove(query, options, (err, result) => {
+      if (err) {
+        reject(err);
+      }
+
+      resolve(result);
+    }));
+  }
+}

src/backend/database/mongodb.ts

@@ -0,0 +1,122 @@
+import { Collection, Filter, MongoClient, OptionalUnlessRequiredId, UpdateFilter } from 'mongodb';
+import { DatabaseDriver, Options } from './types.js';
+import appConfig from '../utils/appConfig.js';
+
+const mongodbUri = appConfig.database.driver === 'mongodb' ? appConfig.database.mongodb.uri : null;
+const mongodbClient = mongodbUri ? await MongoClient.connect(mongodbUri): null;
+
+/**
+ * MongoDB driver for working with database
+ */
+export default class MongoDatabaseDriver<DocType> implements DatabaseDriver<DocType> {
+  /**
+   * Mongo client instance
+   */
+  private db: MongoClient;
+
+  /**
+   * Collection instance
+   */
+  private collection: Collection<DocType>;
+
+  /**
+   * Creates driver instance
+   *
+   * @param collectionName - collection to work with
+   */
+  constructor(collectionName: string) {
+    if (!mongodbClient) {
+      throw new Error('MongoDB client is not initialized');
+    }
+    this.db = mongodbClient;
+    this.collection = mongodbClient.db().collection(collectionName);
+  }
+
+  /**
+   * Insert new document into the database
+   *
+   * @param {object} doc - object to insert
+   * @returns {Promise<object | Error>} - inserted doc or Error object
+   */
+  public async insert(doc: DocType): Promise<DocType> {
+    const result = await this.collection.insertOne(doc as OptionalUnlessRequiredId<DocType>);
+
+    return {
+      ...doc,
+      _id: result.insertedId,
+    };
+  }
+
+  /**
+   * Find documents that match passed query
+   *
+   * @param {object} query - query object
+   * @param {object} projection - projection object
+   * @returns {Promise<Array<object> | Error>} - found docs or Error object
+   */
+  public async find(query: Record<string, unknown>, projection?: DocType): Promise<Array<DocType>> {
+    const cursor = this.collection.find(query as Filter<DocType>);
+
+    if (projection) {
+      cursor.project(projection);
+    }
+
+    const docs = await cursor.toArray();
+
+    return docs as unknown as Array<DocType>;
+  }
+
+  /**
+   * Find one document matches passed query
+   *
+   * @param {object} query - query object
+   * @param {object} projection - projection object
+   * @returns {Promise<object | Error>} - found doc or Error object
+   */
+  public async findOne(query: Record<string, unknown>, projection?: DocType): Promise<DocType> {
+    const doc = await this.collection.findOne(query as Filter<DocType>, { projection });
+
+    return doc as unknown as DocType;
+  }
+
+  /**
+   * Update document matches query
+   *
+   * @param {object} query - query object
+   * @param {object} update - fields to update
+   * @param {Options} options - optional params
+   * @returns {Promise<number | object | object[] | Error>} - number of updated rows or affected docs or Error object
+   */
+  public async update(query: Record<string, unknown>, update: DocType, options: Options = {}): Promise<number|boolean|Array<DocType>> {
+    const updateDocument = {
+      $set: update,
+    } as UpdateFilter<DocType>;
+    const result = await this.collection.updateMany(query as Filter<DocType>, updateDocument, options);
+
+    switch (true) {
+      case options.returnUpdatedDocs:
+        return result.modifiedCount;
+      case options.upsert:
+        if (result.modifiedCount) {
+          return result.modifiedCount;
+        }
+
+        return result as DocType[];
+      default:
+        return result as DocType[];
+    }
+  }
+
+  /**
+   * Remove document matches passed query
+   *
+   * @param {object} query - query object
+   * @param {Options} options - optional params
+   * @returns {Promise<number|Error>} - number of removed rows or Error object
+   */
+  public async remove(query: Record<string, unknown>, options: Options = {}): Promise<number> {
+    const result = await this.collection.deleteMany(query as Filter<DocType>);
+
+    return result.deletedCount;
+  }
+}

src/backend/database/types.ts

@@ -0,0 +1,70 @@
+import { ObjectId } from 'mongodb';
+
+/**
+ * Represents database driver functionality
+ */
+export interface DatabaseDriver<DocType> {
+  /**
+   * Insert new document into the database
+   *
+   * @param {object} doc - object to insert
+   * @returns {Promise<object | Error>} - inserted doc or Error object
+   */
+  insert(doc: DocType): Promise<DocType>;
+
+  /**
+   * Find documents that match passed query
+   *
+   * @param {object} query - query object
+   * @param {object} projection - projection object
+   * @returns {Promise<Array<object> | Error>} - found docs or Error object
+   */
+  find(query: Record<string, unknown>, projection?: DocType): Promise<Array<DocType>>;
+
+  /**
+   * Find one document matches passed query
+   *
+   * @param {object} query - query object
+   * @param {object} projection - projection object
+   * @returns {Promise<object | Error>} - found doc or Error object
+   */
+  findOne(query: Record<string, unknown>, projection?: DocType): Promise<DocType>;
+
+  /**
+   * Update document matches query
+   *
+   * @param {object} query - query object
+   * @param {object} update - fields to update
+   * @param {Options} options - optional params
+   * @returns {Promise<number | object | object[] | Error>} - number of updated rows or affected docs or Error object
+   */
+  update(query: Record<string, unknown>, update: DocType, options: Options): Promise<number|boolean|Array<DocType>>
+
+  /**
+   * Remove document matches passed query
+   *
+   * @param {object} query - query object
+   * @param {Options} options - optional params
+   * @returns {Promise<number|Error>} - number of removed rows or Error object
+   */
+  remove(query: Record<string, unknown>, options: Options): Promise<number>
+}
+
+/**
+ * Represents unique database entity id
+ * unique symbol to prevent type widening (read more https://todayilearned.net/2022/07/typescript-primitive-type-aliases-unique-symbols)
+ */
+export type EntityId = (string | ObjectId) & {readonly id: unique symbol};
+
+/**
+ * @typedef Options - optional params
+ * @param {boolean} multi - (false) allows to take action to several documents
+ * @param {boolean} upsert - (false) if true, upsert document with update fields.
+ *                           Method will return inserted doc or number of affected docs if doc hasn't been inserted
+ * @param {boolean} returnUpdatedDocs - (false) if true, returns affected docs
+ */
+export interface Options {
+  multi?: boolean;
+  upsert?: boolean;
+  returnUpdatedDocs?: boolean;
+}