[Feature] Static pages rendering 🤩 (#274)

src/backend/app.ts

@@ -1,99 +1,20 @@
-import express, { NextFunction, Request, Response } from 'express';
-import path from 'path';
-import { fileURLToPath } from 'url';
-import cookieParser from 'cookie-parser';
-import morgan from 'morgan';
-import routes from './routes/index.js';
-import HttpException from './exceptions/httpException.js';
-import * as dotenv from 'dotenv';
-import HawkCatcher from '@hawk.so/nodejs';
-import os from 'os';
-import { downloadFavicon, FaviconData } from './utils/downloadFavicon.js';
-import appConfig from './utils/appConfig.js';
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import runHttpServer from './server.js';
+import buildStatic from './build-static.js';
 
-/**
- * The __dirname CommonJS variables are not available in ES modules.
- * https://nodejs.org/api/esm.html#no-__filename-or-__dirname
- */
-// eslint-disable-next-line @typescript-eslint/naming-convention
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-dotenv.config();
-const app = express();
-const localConfig = appConfig.frontend;
-
-// Initialize the backend error tracking catcher.
-if (appConfig.hawk?.backendToken) {
-  HawkCatcher.init(appConfig.hawk.backendToken);
-}
-
-// Get url to upload favicon from config
-const favicon = appConfig.favicon;
-
-app.locals.config = localConfig;
-// Set client error tracking token as app local.
-if (appConfig.hawk?.frontendToken) {
-  app.locals.config.hawkClientToken = appConfig.hawk.frontendToken;
-}
-
-// view engine setup
-app.set('views', path.join(__dirname, './', 'views'));
-app.set('view engine', 'twig');
-import('./utils/twig.js');
-
-const downloadedFaviconFolder = os.tmpdir();
-
-// Check if favicon is not empty
-if (favicon) {
-  // Upload favicon by url, it's path on server is '/temp/favicon.{format}'
-  downloadFavicon(favicon, downloadedFaviconFolder).then((res) => {
-    app.locals.favicon = res;
-    console.log('Favicon successfully uploaded');
+yargs(hideBin(process.argv))
+  .option('config', {
+    alias: 'c',
+    type: 'string',
+    default: './docs-config.yaml',
+    description: 'Config files paths',
   })
-    .catch( (err) => {
-      console.log(err);
-      console.log('Favicon has not uploaded');
-    });
-} else {
-  console.log('Favicon is empty, using default path');
-  app.locals.favicon = {
-    destination: '/favicon.png',
-    type: 'image/png',
-  } as FaviconData;
-}
-
-app.use(morgan('dev'));
-app.use(express.json());
-app.use(express.urlencoded({ extended: true }));
-app.use(cookieParser());
-app.use(express.static(path.join(__dirname, '../../public')));
-
-if (appConfig.uploads.driver === 'local') {
-  app.use('/uploads', express.static(appConfig.uploads.local.path));
-}
-
-app.use('/favicon', express.static(downloadedFaviconFolder));
-
-app.use('/', routes);
-
-
-// global error handler
-app.use(function (err: unknown, req: Request, res: Response, next: NextFunction) {
-  // send any type of error to hawk server.
-  if (appConfig.hawk?.backendToken && err instanceof Error) {
-    HawkCatcher.send(err);
-  }
-  // only send Http based exception to client.
-  if (err instanceof HttpException) {
-    // set locals, only providing error in development
-    res.locals.message = err.message;
-    res.locals.error = req.app.get('env') === 'development' ? err : {};
-    // render the error page
-    res.status(err.status || 500);
-    res.render('error');
-  }
-  next(err);
-});
-
-
-export default app;
+  .help('h')
+  .alias('h', 'help')
+  .command('$0', 'start the server', () => {/* empty */}, runHttpServer)
+  .command('build-static', 'build files from database', () => {/* empty */}, async () => {
+    await buildStatic();
+    process.exit(0);
+  })
+  .parse();

src/backend/build-static.ts

@@ -0,0 +1,125 @@
+import twig from 'twig';
+import Page from './models/page.js';
+import PagesFlatArray from './models/pagesFlatArray.js';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import('./utils/twig.js');
+import fs from 'fs/promises';
+import mkdirp from 'mkdirp';
+import { createMenuTree } from './utils/menu.js';
+import { EntityId } from './database/types.js';
+import PagesOrder from './controllers/pagesOrder.js';
+import fse from 'fs-extra';
+import appConfig  from './utils/appConfig.js';
+import Aliases from './controllers/aliases.js';
+import Pages from './controllers/pages.js';
+
+/**
+ * Build static pages from database
+ */
+export default async function buildStatic(): Promise<void> {
+  const config = appConfig.staticBuild;
+
+  if (!config) {
+    throw new Error('Static build config not found');
+  }
+
+  const dirname = path.dirname(fileURLToPath(import.meta.url));
+  const cwd = process.cwd();
+  const distPath = path.resolve(cwd, config.outputDir);
+
+  /**
+   * Render template with twig by path
+   *
+   * @param filePath - path to template
+   * @param data - data to render template
+   */
+  function renderTemplate(filePath: string, data: Record<string, unknown>): Promise<string> {
+    return new Promise((resolve, reject) => {
+      twig.renderFile(path.resolve(dirname, filePath), data, (err, html) => {
+        if (err) {
+          reject(err);
+        }
+        resolve(html);
+      });
+    });
+  }
+
+  console.log('Removing old static files');
+  await fse.remove(distPath);
+
+  console.log('Building static files');
+  const pagesOrder = await PagesOrder.getAll();
+  const allPages = await Page.getAll();
+
+  await mkdirp(distPath);
+
+  /**
+   * Renders single page
+   *
+   * @param page - page to render
+   * @param isIndex - is this page index page
+   */
+  async function renderPage(page: Page, isIndex?: boolean): Promise<void> {
+    console.log(`Rendering page ${page.uri}`);
+    const pageParent = await page.getParent();
+    const pageId = page._id;
+
+    if (!pageId) {
+      throw new Error('Page id is not defined');
+    }
+    const parentIdOfRootPages = '0' as EntityId;
+    const previousPage = await PagesFlatArray.getPageBefore(pageId);
+    const nextPage = await PagesFlatArray.getPageAfter(pageId);
+    const menu = createMenuTree(parentIdOfRootPages, allPages, pagesOrder, 2);
+    const result = await renderTemplate('./views/pages/page.twig', {
+      page,
+      pageParent,
+      previousPage,
+      nextPage,
+      menu,
+      config: appConfig.frontend,
+    });
+
+    const filename = (isIndex || page.uri === '') ? 'index.html' : `${page.uri}.html`;
+
+    await fs.writeFile(path.resolve(distPath, filename), result);
+    console.log(`Page ${page.uri} rendered`);
+  }
+
+  /**
+   * Render index page
+   *
+   * @param indexPageUri - uri of index page
+   */
+  async function renderIndexPage(indexPageUri: string): Promise<void> {
+    const alias = await Aliases.get(indexPageUri);
+
+    if (!alias.id) {
+      throw new Error(`Alias ${indexPageUri} not found`);
+    }
+
+    const page = await Pages.get(alias.id);
+
+    await renderPage(page, true);
+  }
+
+  /**
+   * Render all pages
+   */
+  for (const page of allPages) {
+    await renderPage(page);
+  }
+
+  await renderIndexPage(config.indexPageUri);
+  console.log('Static files built');
+
+  console.log('Copy public directory');
+  await fse.copy(path.resolve(dirname, '../../public'), distPath);
+
+  if (appConfig.uploads.driver === 'local') {
+    console.log('Copy uploads directory');
+    await fse.copy(path.resolve(cwd, appConfig.uploads.local.path), path.resolve(distPath, 'uploads'));
+  }
+}
+

src/backend/routes/middlewares/pages.ts

@@ -1,57 +1,10 @@
 import { NextFunction, Request, Response } from 'express';
 import Pages from '../../controllers/pages.js';
 import PagesOrder from '../../controllers/pagesOrder.js';
-import Page from '../../models/page.js';
 import asyncMiddleware from '../../utils/asyncMiddleware.js';
-import PageOrder from '../../models/pageOrder.js';
 import { EntityId } from '../../database/types.js';
-import { isEqualIds } from '../../database/index.js';
+import { createMenuTree } from '../../utils/menu.js';
 
-/**
- * Process one-level pages list to parent-children list
- *
- * @param {string} parentPageId - parent page id
- * @param {Page[]} pages - list of all available pages
- * @param {PagesOrder[]} pagesOrder - list of pages order
- * @param {number} level - max level recursion
- * @param {number} currentLevel - current level of element
- * @returns {Page[]}
- */
-function createMenuTree(parentPageId: EntityId, pages: Page[], pagesOrder: PageOrder[], level = 1, currentLevel = 1): Page[] {
-  const childrenOrder = pagesOrder.find(order => isEqualIds(order.data.page, parentPageId));
-
-  /**
-   * branch is a page children in tree
-   * if we got some children order on parents tree, then we push found pages in order sequence
-   * otherwise just find all pages includes parent tree
-   */
-  let ordered: any[] = [];
-
-  if (childrenOrder) {
-    ordered = childrenOrder.order.map((pageId: EntityId) => {
-      return pages.find(page => isEqualIds(page._id, pageId));
-    });
-  }
-
-  const unordered = pages.filter(page => isEqualIds(page._parent, parentPageId));
-  const branch = Array.from(new Set([...ordered, ...unordered]));
-
-  /**
-   * stop recursion when we got the passed max level
-   */
-  if (currentLevel === level + 1) {
-    return [];
-  }
-
-  /**
-   * Each parents children can have subbranches
-   */
-  return branch.filter(page => page && page._id).map(page => {
-    return Object.assign({
-      children: createMenuTree(page._id, pages, pagesOrder, level, currentLevel + 1),
-    }, page.data);
-  });
-}
 
 /**
  * Middleware for all /page/... routes

src/backend/server.ts

@@ -0,0 +1,210 @@
+#!/usr/bin/env node
+/**
+ * Module dependencies.
+ */
+import http from 'http';
+import Debug from 'debug';
+import appConfig from './utils/appConfig.js';
+import { drawBanner } from './utils/banner.js';
+import express, { NextFunction, Request, Response } from 'express';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import HawkCatcher from '@hawk.so/nodejs';
+import os from 'os';
+import { downloadFavicon, FaviconData } from './utils/downloadFavicon.js';
+import morgan from 'morgan';
+import cookieParser from 'cookie-parser';
+import routes from './routes/index.js';
+import HttpException from './exceptions/httpException.js';
+
+const debug = Debug.debug('codex.docs:server');
+
+/**
+ * Get port from environment and store in Express.
+ */
+const port = normalizePort(appConfig.port.toString() || '3000');
+
+/**
+ * Create Express server
+ */
+function createApp(): express.Express {
+  /**
+   * The __dirname CommonJS variables are not available in ES modules.
+   * https://nodejs.org/api/esm.html#no-__filename-or-__dirname
+   */
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+  const app = express();
+  const localConfig = appConfig.frontend;
+
+  // Initialize the backend error tracking catcher.
+  if (appConfig.hawk?.backendToken) {
+    HawkCatcher.init(appConfig.hawk.backendToken);
+  }
+
+  // Get url to upload favicon from config
+  const favicon = appConfig.favicon;
+
+  app.locals.config = localConfig;
+  // Set client error tracking token as app local.
+  if (appConfig.hawk?.frontendToken) {
+    app.locals.config.hawkClientToken = appConfig.hawk.frontendToken;
+  }
+
+  // view engine setup
+  app.set('views', path.join(__dirname, './', 'views'));
+  app.set('view engine', 'twig');
+  import('./utils/twig.js');
+
+  const downloadedFaviconFolder = os.tmpdir();
+
+  // Check if favicon is not empty
+  if (favicon) {
+    // Upload favicon by url, it's path on server is '/temp/favicon.{format}'
+    downloadFavicon(favicon, downloadedFaviconFolder).then((res) => {
+      app.locals.favicon = res;
+      console.log('Favicon successfully uploaded');
+    })
+      .catch((err) => {
+        console.log(err);
+        console.log('Favicon has not uploaded');
+      });
+  } else {
+    console.log('Favicon is empty, using default path');
+    app.locals.favicon = {
+      destination: '/favicon.png',
+      type: 'image/png',
+    } as FaviconData;
+  }
+
+  app.use(morgan('dev'));
+  app.use(express.json());
+  app.use(express.urlencoded({ extended: true }));
+  app.use(cookieParser());
+  app.use(express.static(path.join(__dirname, '../../public')));
+
+  if (appConfig.uploads.driver === 'local') {
+    app.use('/uploads', express.static(appConfig.uploads.local.path));
+  }
+
+  app.use('/favicon', express.static(downloadedFaviconFolder));
+
+  app.use('/', routes);
+
+
+  // global error handler
+  app.use(function (err: unknown, req: Request, res: Response, next: NextFunction) {
+    // send any type of error to hawk server.
+    if (appConfig.hawk?.backendToken && err instanceof Error) {
+      HawkCatcher.send(err);
+    }
+    // only send Http based exception to client.
+    if (err instanceof HttpException) {
+      // set locals, only providing error in development
+      res.locals.message = err.message;
+      res.locals.error = req.app.get('env') === 'development' ? err : {};
+      // render the error page
+      res.status(err.status || 500);
+      res.render('error');
+    }
+    next(err);
+  });
+
+  return app;
+}
+
+/**
+ * Create and run HTTP server.
+ */
+export default function runHttpServer(): void {
+  const app = createApp();
+
+  app.set('port', port);
+
+  /**
+   * Create HTTP server.
+   */
+  const server = http.createServer(app);
+
+  /**
+   * Event listener for HTTP server 'listening' event.
+   */
+  function onListening(): void {
+    const addr = server.address();
+
+    if (addr === null) {
+      debug('Address not found');
+      process.exit(1);
+    }
+
+    const bind = typeof addr === 'string'
+      ? 'pipe ' + addr
+      : 'port ' + addr.port;
+
+    debug('Listening on ' + bind);
+
+    drawBanner([
+      `CodeX Docs server is running`,
+      ``,
+      `Main page: http://localhost:${port}`,
+    ]);
+  }
+
+  /**
+   * Listen on provided port, on all network interfaces.
+   */
+  server.listen(port);
+  server.on('error', onError);
+  server.on('listening', onListening);
+}
+
+/**
+ * Normalize a port into a number, string, or false.
+ *
+ * @param val
+ */
+function normalizePort(val: string): number | string | false {
+  const value = parseInt(val, 10);
+
+  if (isNaN(value)) {
+    // named pipe
+    return val;
+  }
+
+  if (value >= 0) {
+    // port number
+    return value;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server 'error' event.
+ *
+ * @param error
+ */
+function onError(error: NodeJS.ErrnoException): void {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  const bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}

src/backend/utils/appConfig.ts

@@ -84,6 +84,16 @@ const FrontendConfig = z.object({
     uri: z.string() })])), // Menu for pages
 });
 
+/**
+ * Static build configuration
+ */
+const StaticBuildConfig = z.object({
+  outputDir: z.string(), // Output directory for static build
+  indexPageUri: z.string(), // URI for index page to render
+});
+
+export type StaticBuildConfig = z.infer<typeof StaticBuildConfig>;
+
 /**
  * Application configuration
  */
@@ -97,6 +107,7 @@ const AppConfig = z.object({
   frontend: FrontendConfig, // Frontend configuration
   auth: AuthConfig, // Auth configuration
   database: z.union([LocalDatabaseConfig, MongoDatabaseConfig]), // Database configuration
+  staticBuild: StaticBuildConfig.optional(), // Static build configuration
 });
 
 export type AppConfig = z.infer<typeof AppConfig>;
@@ -107,7 +118,7 @@ const args = arg({ /* eslint-disable @typescript-eslint/naming-convention */
 });
 
 const cwd = process.cwd();
-const paths = (args['--config'] || [ './app-config.yaml' ]).map((configPath) => {
+const paths = (args['--config'] || [ './docs-config.yaml' ]).map((configPath) => {
   if (path.isAbsolute(configPath)) {
     return configPath;
   }

src/backend/utils/banner.ts

@@ -0,0 +1,33 @@
+/**
+ * Draw banner in console with given text lines
+ *
+ * @param lines - data to draw
+ */
+export function drawBanner(lines: string[]): void {
+  /** Define banner parts */
+  const PARTS = {
+    TOP_LEFT: '┌',
+    TOP_RIGHT: '┐',
+    BOTTOM_LEFT: '└',
+    BOTTOM_RIGHT: '┘',
+    HORIZONTAL: '─',
+    VERTICAL: '│',
+    SPACE: ' ',
+  };
+
+  /** Calculate max line length */
+  const maxLength = lines.reduce((max, line) => Math.max(max, line.length), 0);
+
+  /** Prepare top line */
+  const top = PARTS.TOP_LEFT + PARTS.HORIZONTAL.repeat(maxLength + 2) + PARTS.TOP_RIGHT;
+
+  /** Compose middle lines */
+  const middle = lines.map(line => PARTS.VERTICAL + ' ' + line + PARTS.SPACE.repeat(maxLength - line.length) + ' ' + PARTS.VERTICAL);
+
+  /** Prepare bottom line */
+  const bottom = PARTS.BOTTOM_LEFT + PARTS.HORIZONTAL.repeat(maxLength + 2) + PARTS.BOTTOM_RIGHT;
+
+  console.log(top);
+  console.log(middle.join('\n'));
+  console.log(bottom);
+}

src/backend/utils/menu.ts

@@ -0,0 +1,49 @@
+import { EntityId } from '../database/types.js';
+import Page from '../models/page.js';
+import PageOrder from '../models/pageOrder.js';
+import { isEqualIds } from '../database/index.js';
+
+/**
+ * Process one-level pages list to parent-children list
+ *
+ * @param parentPageId - parent page id
+ * @param pages - list of all available pages
+ * @param pagesOrder - list of pages order
+ * @param level - max level recursion
+ * @param currentLevel - current level of element
+ */
+export function createMenuTree(parentPageId: EntityId, pages: Page[], pagesOrder: PageOrder[], level = 1, currentLevel = 1): Page[] {
+  const childrenOrder = pagesOrder.find(order => isEqualIds(order.data.page, parentPageId));
+
+  /**
+   * branch is a page children in tree
+   * if we got some children order on parents tree, then we push found pages in order sequence
+   * otherwise just find all pages includes parent tree
+   */
+  let ordered: any[] = [];
+
+  if (childrenOrder) {
+    ordered = childrenOrder.order.map((pageId: EntityId) => {
+      return pages.find(page => isEqualIds(page._id, pageId));
+    });
+  }
+
+  const unordered = pages.filter(page => isEqualIds(page._parent, parentPageId));
+  const branch = Array.from(new Set([...ordered, ...unordered]));
+
+  /**
+   * stop recursion when we got the passed max level
+   */
+  if (currentLevel === level + 1) {
+    return [];
+  }
+
+  /**
+   * Each parents children can have subbranches
+   */
+  return branch.filter(page => page && page._id).map(page => {
+    return Object.assign({
+      children: createMenuTree(page._id, pages, pagesOrder, level, currentLevel + 1),
+    }, page.data);
+  });
+}

src/backend/views/pages/index.twig

@@ -12,13 +12,13 @@
 <script>
 </script>
 <body class="greeting-body">
-  {% include "components/header.twig" %}
+  {% include "../components/header.twig" %}
   <div class="greeting-content">
     {{ svg('frog') }}
     <p class="greeting-content__message">
       It’s time to create the first page!
     </p>
-    {% include 'components/button.twig' with {label: 'Add page', icon: 'plus', size: 'small', url: '/page/new'} %}
+    {% include '../components/button.twig' with {label: 'Add page', icon: 'plus', size: 'small', url: '/page/new'} %}
   </div>
   {% if config.yandexMetrikaId is not empty %}
     <script type="text/javascript" >

src/backend/views/pages/page.twig

@@ -1,4 +1,4 @@
-{% extends 'layout.twig' %}
+{% extends '../layout.twig' %}
 
 {% block body %}
   <article class="page" data-module="page">
@@ -44,7 +44,7 @@
         {% endif %}
       {%  endfor %}
     </section>
-    {% include 'components/navigator.twig' with {previousPage:  previousPage, nextPage: nextPage} %}
+    {% include '../components/navigator.twig' with {previousPage:  previousPage, nextPage: nextPage} %}
   </article>
 
 {% endblock %}