search implementation

nodemon.json

@@ -9,5 +9,5 @@
   "watch": [
     "**/*"
   ],
-  "ext": "js,twig"
+  "ext": "ts,js,twig"
 }

package.json

@@ -45,6 +45,7 @@
     "@babel/polyfill": "^7.12.1",
     "@babel/preset-env": "^7.16.11",
     "@codexteam/misprints": "^1.0.0",
+    "@codexteam/shortcuts": "^1.2.0",
     "@editorjs/checklist": "^1.3.0",
     "@editorjs/code": "^2.7.0",
     "@editorjs/delimiter": "^1.2.0",
@@ -81,6 +82,7 @@
     "@types/sinon": "^10.0.2",
     "@types/twig": "^1.12.6",
     "autoprefixer": "^10.4.2",
+    "axios": "^0.27.2",
     "babel": "^6.23.0",
     "babel-eslint": "^10.0.1",
     "babel-loader": "^8.2.3",

src/backend/controllers/search.ts

@@ -0,0 +1,128 @@
+import Page from '../models/page';
+import Pages from '../controllers/pages';
+
+type SearchResponse = {
+  completions: string[];
+
+  pages: Page[];
+}
+
+class Search {
+  /**
+   * Prepare words database
+   */
+  public async index() {
+    /**
+     * Prepare pages content for the search
+     * @todo - it should be done in the background
+     */
+    const pages = await Pages.getAll();
+    const pagesWords = pages.map(page => {
+      const pageWords: string[] = [];
+
+      page.body.blocks.forEach((block: any) => {
+        let blockContent = '';
+
+        const validBlocks = ['header', 'paragraph'];
+        if (!validBlocks.includes(block.type)) {
+          return;
+        }
+
+        switch (block.type) {
+          case 'header':
+            blockContent = block.data.text;
+            break;
+
+          case 'paragraph':
+            blockContent = block.data.text
+            break;
+        }
+
+        const blockWords: string[] = blockContent
+          // @todo get text from inline code elements and remove html tags
+
+          // left only letters and numbers
+          .replace(/[^a-z0-9]/gi, ' ')
+
+          // lowercase all words
+          .toLowerCase()
+
+          // remove multiple spaces
+          .replace(/\s+/g, ' ')
+
+          // split to words by spaces
+          .split(' ');
+
+        pageWords.push(...blockWords);
+      });
+
+      const uniqueWords = [...new Set(pageWords)].sort();
+
+      return {
+        id: page._id,
+        words: uniqueWords
+      };
+    });
+
+    return pagesWords;
+  }
+
+  public async query(searchString: string): Promise<SearchResponse> {
+    const pages = await Pages.getAll();
+    const pagesWords = await this.index();
+
+    /**
+     * Search itself
+     */
+    const searchWords = searchString.toLowerCase().split(' ');
+    const goodPages = pagesWords.map(({ id, words}) => {
+      const foundWords = searchWords.filter(
+        word => {
+          return words.filter(
+            testWord => {
+              return testWord.indexOf(word) === 0
+            }
+          ).length > 0;
+        }
+      );
+
+      const successRatio = foundWords.length / searchWords.length * 100;
+
+      return {
+        id,
+        successRatio
+      }
+    });
+
+    const foundPages = goodPages
+      .filter(({ successRatio }) => successRatio > 50)
+      .sort((a, b) => b.successRatio - a.successRatio)
+      .slice(0, 10);
+
+
+    // --------- START test ---------
+
+    const uniqWords = [...new Set(pagesWords.flatMap(page => page.words))].sort();
+
+    uniqWords.forEach(word => {
+      console.log(word);
+    })
+
+    // --------- END test ---------
+
+
+
+    return {
+      completions: uniqWords.filter(word => word.indexOf(searchWords.slice(-1)[0]) === 0),
+      pages: pages.filter(page => foundPages.some(({ id }) => id === page._id))
+    }
+  }
+
+  private async search(searchString: string) {
+    const pages = await this.query(searchString);
+
+    return pages;
+  }
+}
+
+export default Search;

src/backend/routes/api/index.ts

@@ -1,12 +1,15 @@
 import express from 'express';
+
 import pagesAPI from './pages';
 import transportAPI from './transport';
 import linksAPI from './links';
+import searchAPI from './search';
 
 const router = express.Router();
 
 router.use('/', pagesAPI);
 router.use('/', transportAPI);
 router.use('/', linksAPI);
+router.use('/', searchAPI);
 
 export default router;

src/backend/routes/api/search.ts

@@ -0,0 +1,54 @@
+import express, { Request, Response } from 'express';
+import Search from '../../controllers/search';
+
+const router = express.Router();
+
+/**
+ * GET /search/:searchString
+ *
+ * Search given words in all documents
+ */
+router.get('/search', async (req: Request, res: Response) => {
+  try {
+    const searchString = req.query.text as string;
+
+    /** Start measuring search time */
+    const startTime = performance.now();
+
+    const search = new Search();
+    const searchResponse = await search.query(searchString);
+
+    /** End measuring search time */
+    const endTime = performance.now();
+
+    /** Show search time */
+    const searchItem = (endTime - startTime).toFixed(6);
+    console.log(`🔎 "${searchString}" ⏱  ${searchItem} ms`);
+
+    const compactedPages = searchResponse.pages.map(page => {
+      return {
+        _id: page._id,
+        title: page.title,
+        uri: page.uri,
+        // body: page.body,
+        // parent: page.parent,
+      };
+    });
+
+    res.json({
+      success: true,
+      result: {
+        completions: searchResponse.completions,
+        pages: compactedPages,
+        time: searchItem,
+      },
+    });
+  } catch (err) {
+    res.status(400).json({
+      success: false,
+      error: (err as Error).message,
+    });
+  }
+});
+
+export default router;

src/backend/views/layout.twig

@@ -15,7 +15,7 @@
         hawkClientToken:"{{ config.hawkClientToken }}",
       };
     </script>
-  <body>
+  <body data-module="search">
     {% include "components/header.twig" with res.locals.isAuthorized %}
     <div class="docs">
 

src/frontend/js/app.js

@@ -17,6 +17,7 @@ import Writing from './modules/writing';
 import Page from './modules/page';
 import Extensions from './modules/extensions';
 import Sidebar from './modules/sidebar';
+import Search from './modules/search';
 import HawkCatcher from '@hawk.so/javascript';
 
 /**
@@ -31,6 +32,8 @@ class Docs {
     this.page = new Page();
     this.extensions = new Extensions();
     this.sidebar = new Sidebar();
+    this.search = new Search();
+
     if (window.config.hawkClientToken) {
       this.hawk = new HawkCatcher(window.config.hawkClientToken);
     }

src/frontend/js/modules/page.js

@@ -17,6 +17,7 @@ export default class Page {
   init() {
     this.codeStyler = this.createCodeStyling();
     this.tableOfContent = this.createTableOfContent();
+    this.search = this.createSearch();
   }
 
   /**

src/frontend/js/modules/search.js

@@ -0,0 +1,137 @@
+import { debounce } from '../utils/decorators';
+import Shortcut from '@codexteam/shortcuts';
+import axios from 'axios';
+
+
+export default class Search {
+  constructor() {
+    this.nodes = {
+      overlay: null,
+      searchWrapper: null,
+      searchInput: null,
+      searchResultsWrapper: null
+    };
+
+    this.isVisible = false;
+
+    this.shortcut = null;
+    this.TOGGLER_SHORTCUT = 'CMD+SHIFT+F';
+
+    this.debouncedSearch = null;
+    this.DEBOUNCE_TIME = 500;
+  }
+
+  init(settings = {}, moduleEl) {
+    console.log('search init');
+
+    this.createSearchOverlay();
+    this.createDebouncedSearch();
+    this.enableShortcutListening();
+
+    // ! force open search overlay
+    // this.toggleSearchOverlay(true);
+  }
+
+  createSearchOverlay() {
+    this.nodes.overlay = document.createElement('div');
+    this.nodes.overlay.classList.add('search-overlay');
+    this.nodes.overlay.addEventListener('click', this.searchOverlayClickProcessor.bind(this));
+
+    this.nodes.searchWrapper = document.createElement('div');
+    this.nodes.searchWrapper.classList.add('search-wrapper');
+
+    this.nodes.searchInput = document.createElement('input');
+    this.nodes.searchInput.classList.add('search-input');
+    this.nodes.searchInput.setAttribute('type', 'search');
+    this.nodes.searchInput.setAttribute('placeholder', 'Find in documents...');
+    this.nodes.searchInput.setAttribute('autocomplete', 'off');
+    this.nodes.searchInput.addEventListener('input', this.searchInputOnchangeProcessor.bind(this));
+    this.nodes.searchWrapper.appendChild(this.nodes.searchInput);
+
+    this.nodes.searchResultsWrapper = document.createElement('div');
+    this.nodes.searchResultsWrapper.classList.add('search-results-wrapper');
+    this.nodes.searchWrapper.appendChild(this.nodes.searchResultsWrapper);
+
+    this.nodes.overlay.appendChild(this.nodes.searchWrapper);
+    document.body.appendChild(this.nodes.overlay);
+  }
+
+  searchOverlayClickProcessor(event) {
+    if (event.target !== this.nodes.overlay) {
+      return;
+    }
+
+    this.toggleSearchOverlay(false);
+  }
+
+  searchInputOnchangeProcessor(event) {
+    // close search overlay if ESC key is pressed
+    if (event.keyCode === 27) {
+      this.toggleSearchOverlay(false);
+      event.preventDefault();
+    }
+
+    console.log(event.target.value);
+
+    this.debouncedSearch(event.target.value);
+  }
+
+  enableShortcutListening() {
+    this.shortcut = new Shortcut({
+      name : this.TOGGLER_SHORTCUT,
+      on : document.body,
+      callback: (event) => {
+        this.toggleSearchOverlay();
+      }
+    });
+  }
+
+  toggleSearchOverlay(force) {
+    this.isVisible = force || !this.isVisible;
+
+    this.nodes.overlay.classList.toggle('search-overlay--visible', this.isVisible);
+    document.body.classList.toggle('noscroll', this.isVisible);
+
+    this.nodes.searchInput.focus();
+  }
+
+  createDebouncedSearch() {
+    this.debouncedSearch = debounce(this.getSearchResults, this.DEBOUNCE_TIME);
+  }
+
+  getSearchResults(text) {
+    // this.showSearchResult(text);
+
+    // call api to get search results
+    axios.get('/api/search', {
+      params: {
+        text: text
+      }
+    })
+      .then(this.showSearchResult.bind(this));
+  }
+
+  showSearchResult({ data }) {
+    console.log(data);
+
+    this.nodes.searchResultsWrapper.innerHTML = '';
+
+
+    data.result.pages.forEach(page => {
+      const result = document.createElement('a');
+      result.classList.add('search-results-item');
+      result.setAttribute('href', `/${page.uri}`);
+
+      const title = document.createElement('div');
+      title.classList.add('search-results-item__title');
+      title.innerHTML = page.title;
+      result.appendChild(title);
+
+      // const description = document.createElement('div');
+      // description.classList.add('search-results-item__description');
+      // result.appendChild(description);
+
+      this.nodes.searchResultsWrapper.appendChild(result);
+    });
+  }
+}

src/frontend/styles/components/search.pcss

@@ -0,0 +1,78 @@
+.search {
+  &-overlay {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    background: rgba(0, 0, 0, 0.5);
+    z-index: 999;
+    display: none;
+    overflow-y: scroll;
+
+    cursor: pointer;
+
+    &--visible {
+      display: block;
+    }
+  }
+
+  &-wrapper {
+    background: #fff;
+    margin: 30vh auto 10vh;
+    max-width: var(--layout-width-main-col);
+    width: 100%;
+
+    @apply --squircle;
+  }
+
+  &-input {
+    width: 100%;
+    background: #bbb;
+    padding: 15px 25px;
+
+    -webkit-appearance: none;
+    -moz-appearance: none;
+    appearance: none;
+    border: none;
+    outline: none;
+    background-color: transparent;
+
+    &::placeholder {
+      font-weight: 500;
+    }
+  }
+
+  &-results {
+    &-wrapper {
+
+    }
+
+    &-item {
+      padding: 15px 25px;
+      display: block;
+
+      &:nth-child(odd) {
+        background: #f5f5f5;
+      }
+
+      &:hover {
+        background: #ddd;
+      }
+
+      &__title {
+        font-weight: 500;
+      }
+
+      &__description {
+      }
+    }
+  }
+
+}
+
+.noscroll {
+  overflow: hidden;
+}
+
+

src/frontend/styles/main.pcss

@@ -12,6 +12,7 @@
 @import './components/sidebar.pcss';
 @import './components/navigator.pcss';
 @import './components/table-of-content.pcss';
+@import './components/search.pcss';
 
 body {
   font-family: system-ui, Helvetica, Arial, Verdana;

yarn.lock

@@ -805,6 +805,11 @@
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/@codexteam/misprints/-/misprints-1.0.0.tgz#e5a7dec7389fe0f176cd51a040d6dc9bdc252086"
 
+"@codexteam/shortcuts@^1.2.0":
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/@codexteam/shortcuts/-/shortcuts-1.2.0.tgz#b8dd7396962b0bd845a5c8f8f19bc6119b520e19"
+  integrity sha512-Udb8lkwhXEiPoLm7krtUv2f8jYQTutHxsLecmsMvMbOxMJ49LA/EUUzn8Fo32mxOFrI7qozOovspLhHb+y60nQ==
+
 "@cspotcode/source-map-consumer@0.8.0":
   version "0.8.0"
   resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"
@@ -1698,6 +1703,14 @@ axios@^0.21.1:
   dependencies:
     follow-redirects "^1.14.0"
 
+axios@^0.27.2:
+  version "0.27.2"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.27.2.tgz#207658cc8621606e586c85db4b41a750e756d972"
+  integrity sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==
+  dependencies:
+    follow-redirects "^1.14.9"
+    form-data "^4.0.0"
+
 babel-eslint@^10.0.1:
   version "10.1.0"
   resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232"
@@ -3057,7 +3070,7 @@ flatten@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/flatten/-/flatten-1.0.3.tgz#c1283ac9f27b368abc1e36d1ff7b04501a30356b"
 
-follow-redirects@^1.14.0:
+follow-redirects@^1.14.0, follow-redirects@^1.14.9:
   version "1.15.1"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5"
   integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==
@@ -3089,6 +3102,15 @@ form-data@^3.0.0:
     combined-stream "^1.0.8"
     mime-types "^2.1.12"
 
+form-data@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
+  integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.8"
+    mime-types "^2.1.12"
+
 formidable@^1.2.0:
   version "1.2.6"
   resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168"
@@ -3247,18 +3269,6 @@ growl@1.10.5:
   version "1.10.5"
   resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e"
 
-handlebars@^4.1.0:
-  version "4.7.7"
-  resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1"
-  integrity sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==
-  dependencies:
-    minimist "^1.2.5"
-    neo-async "^2.6.0"
-    source-map "^0.6.1"
-    wordwrap "^1.0.0"
-  optionalDependencies:
-    uglify-js "^3.1.4"
-
 has-bigints@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.1.tgz#64fe6acb020673e3b78db035a5af69aa9d07b113"
@@ -3305,6 +3315,11 @@ hosted-git-info@^2.1.4:
   version "2.8.9"
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
 
+html-escaper@^2.0.0:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453"
+  integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==
+
 htmlparser2@^6.1.0:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7"
@@ -3920,7 +3935,7 @@ make-dir@^1.3.0:
   dependencies:
     pify "^3.0.0"
 
-make-dir@^2.0.0:
+make-dir@^2.0.0, make-dir@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5"
   dependencies:
@@ -4147,7 +4162,7 @@ negotiator@0.6.3:
   version "0.6.3"
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd"
 
-neo-async@^2.6.0, neo-async@^2.6.2:
+neo-async@^2.6.2:
   version "2.6.2"
   resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
   integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
@@ -5597,7 +5612,7 @@ supports-color@^5.3.0, supports-color@^5.4.0:
   dependencies:
     has-flag "^3.0.0"
 
-supports-color@^6.0.0:
+supports-color@^6.1.0:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3"
   integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==
@@ -5829,11 +5844,6 @@ typescript@^4.3.5:
   version "4.6.3"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.3.tgz#eefeafa6afdd31d725584c67a0eaba80f6fc6c6c"
 
-uglify-js@^3.1.4:
-  version "3.16.3"
-  resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.16.3.tgz#94c7a63337ee31227a18d03b8a3041c210fd1f1d"
-  integrity sha512-uVbFqx9vvLhQg0iBaau9Z75AxWJ8tqM9AV890dIZCLApF4rTcyHwmAvLeEdYRs+BzYWu8Iw81F79ah0EfTXbaw==
-
 uid-safe@2.1.5:
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/uid-safe/-/uid-safe-2.1.5.tgz#2b3d5c7240e8fc2e58f8aa269e5ee49c0857bd3a"
@@ -6044,11 +6054,6 @@ word-wrap@^1.2.3:
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
 
-wordwrap@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb"
-  integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==
-
 wrap-ansi@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"