feat(backend): ✨ start multi-tenant support (WIP) (#680)

* fix ts types

* feat(code-generation): ♻️ update code-generation formats

* new scope

* add step button

* fix linter error

* update code-generation tags

* feat(backend): ✨ start multi-tenant support

* feat(backend): ✨ group invitation token generation and signup

* refactor(backend): ♻️ move group admin actions to admin router

* set url base to include `/admin`

* feat(frontend): ✨ generate user sign-up links

* test(backend): ✅ refactor test-suite to further decouple tests (WIP)

* feat(backend): 🐛 assign owner on backup import for recipes

* fix(backend): 🐛 assign recipe owner on migration from other service

Co-authored-by: hay-kot <hay-kot@pm.me>

dev/code-generation/_gen_utils.py

@@ -0,0 +1,95 @@
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Tuple
+
+from black import FileMode, format_str
+from jinja2 import Template
+
+
+def render_python_template(template_file: Path, dest: Path, data: dict) -> str:
+    """ Render and Format a Jinja2 Template for Python Code"""
+    tplt = Template(template_file.read_text())
+    text = tplt.render(data)
+    text = format_str(text, mode=FileMode())
+    dest.write_text(text)
+
+
+@dataclass
+class CodeSlicer:
+    start: int
+    end: int
+
+    indentation: str
+    text: list[str]
+
+    _next_line = None
+
+    def purge_lines(self) -> None:
+        start = self.start + 1
+        end = self.end
+        del self.text[start:end]
+
+    def push_line(self, string: str) -> None:
+        self._next_line = self._next_line or self.start + 1
+        print(self.indentation)
+        self.text.insert(self._next_line, self.indentation + string + "\n")
+        self._next_line += 1
+
+
+def get_indentation_of_string(line: str, comment_char: str = "//") -> str:
+    return re.sub(rf"{comment_char}.*", "", line).removesuffix("\n")
+
+
+def find_start_end(file_text: list[str], gen_id: str) -> Tuple[int, int]:
+    start = None
+    end = None
+    indentation = None
+
+    for i, line in enumerate(file_text):
+        if "CODE_GEN_ID:" in line and gen_id in line:
+            start = i
+            indentation = get_indentation_of_string(line)
+        if f"END: {gen_id}" in line:
+            end = i
+
+    if start is None or end is None:
+        raise Exception("Could not find start and end of code generation block")
+
+    if start > end:
+        raise Exception(f"Start ({start=}) of code generation block is after end ({end=})")
+
+    return start, end, indentation
+
+
+def inject_inline(file_path: Path, key: str, code: list[str]) -> None:
+    """Injects a list of strings into the file where the key is found in the format defined
+    by the code-generation. Strings are properly indented and a '\n' is added to the end of
+    each string.
+
+    Start -> 'CODE_GEN_ID: <key>'
+    End -> 'END: <key>'
+
+    If no 'CODE_GEN_ID: <key>' is found, and exception is raised
+
+    Args:
+        file_path (Path): Write to file
+        key (str): CODE_GEN_ID: <key>
+        code (list[str]): List of strings to inject.
+
+    """
+
+    with open(file_path, "r") as f:
+        file_text = f.readlines()
+
+    start, end, indentation = find_start_end(file_text, key)
+
+    slicer = CodeSlicer(start, end, indentation, file_text)
+
+    slicer.purge_lines()
+
+    for line in code:
+        slicer.push_line(line)
+
+    with open(file_path, "w") as file:
+        file.writelines(slicer.text)

dev/code-generation/_open_api_parser.py

@@ -0,0 +1,131 @@
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from _static import Directories
+from fastapi import FastAPI
+from humps import camelize
+from slugify import slugify
+
+
+def get_openapi_spec_by_ref(app, type_reference: str) -> dict:
+    if not type_reference:
+        return None
+
+    schemas = app["components"]["schemas"]
+    type_text = type_reference.split("/")[-1]
+    return schemas.get(type_text, type_reference)
+
+
+def recursive_dict_search(data: dict[str, Any], key: str) -> Any:
+    """
+    Walks a dictionary searching for a key and returns all the keys
+    matching the provided key"""
+    if key in data:
+        return data[key]
+    for _, v in data.items():
+        if isinstance(v, dict):
+            result = recursive_dict_search(v, key)
+            if result:
+                return result
+    return None
+
+
+class APIFunction:
+    def __init__(self, app, route: str, verb: str, data: dict):
+        self.name_camel = camelize(data.get("summary"))
+        self.name_snake = slugify(data.get("summary"), separator="_")
+
+        self.http_verb = verb
+        self.path_vars = re.findall(r"\{(.*?)\}", route)
+        self.path_is_func = "{" in route
+        self.js_route = route.replace("{", "${")
+        self.py_route = route
+
+        self.body_schema = get_openapi_spec_by_ref(app, recursive_dict_search(data, "$ref"))
+
+    def path_args(self) -> str:
+        return ", ".join(x + ": string | number" for x in self.path_vars)
+
+    # body: Optional[list[str]] = []
+    # path_params: Optional[list[str]] = []
+    # query_params: Optional[list[str]] = []
+
+
+# class APIModule(BaseModel):
+#     name: str
+#     functions: list[APIFunction]
+
+
+class OpenAPIParser:
+    def __init__(self, app: FastAPI) -> None:
+        self.app = app
+        self.spec = app.openapi()
+
+        self.modules = {}
+
+    def dump(self, out_path: Path) -> Path:
+        """ Writes the Open API as JSON to a json file"""
+        OPEN_API_FILE = out_path or Directories.out_dir / "openapi.json"
+
+        with open(OPEN_API_FILE, "w") as f:
+            f.write(json.dumps(self.spec, indent=4))
+
+    def _group_by_module_tag(self):
+        """
+        Itterates over all routes and groups them by module. Modules are determined
+        by the suffix text before : in the first tag for the router. These are used
+        to generate the typescript class interface for interacting with the API
+        """
+        modules = {}
+
+        all_paths = self.spec["paths"]
+        for path, http_verbs in all_paths.items():
+            for _, value in http_verbs.items():
+                if "tags" in value:
+                    tag: str = value["tags"][0]
+                    if ":" in tag:
+                        tag = tag.removeprefix('"').split(":")[0].replace(" ", "")
+                        if modules.get(tag):
+                            modules[tag][path] = http_verbs
+                        else:
+                            modules[tag] = {path: http_verbs}
+
+        return modules
+
+    def _get_openapi_spec(self, type_reference: str) -> dict:
+        schemas = self.app["components"]["schemas"]
+        type_text = type_reference.split("/")[-1]
+        return schemas.get(type_text, type_reference)
+
+    def _fill_schema_references(self, raw_modules: dict) -> dict:
+        for _, routes in raw_modules.items():
+            for _, verbs in routes.items():
+                for _, value in verbs.items():
+                    if "requestBody" in value:
+                        try:
+                            schema_ref = value["requestBody"]["content"]["application/json"]["schema"]["$ref"]
+                            schema = self._get_openapi_spec(schema_ref)
+                            value["requestBody"]["content"]["application/json"]["schema"] = schema
+                        except Exception:
+                            continue
+
+        return raw_modules
+
+    def get_by_module(self) -> dict:
+        """Returns paths where tags are split by : and left right is considered the module"""
+        raw_modules = self._group_by_module_tag()
+
+        modules = {}
+        for module_name, routes in raw_modules.items():
+            for route, verbs in routes.items():
+                for verb, value in verbs.items():
+                    function = APIFunction(self.spec, route, verb, value)
+
+                    if modules.get(module_name):
+                        modules[module_name].append(function)
+                    else:
+                        modules[module_name] = [function]
+
+        return modules

dev/code-generation/_router.py

@@ -0,0 +1,86 @@
+import re
+from enum import Enum
+from typing import Optional
+
+from humps import camelize
+from pydantic import BaseModel, Field
+from slugify import slugify
+
+
+class RouteObject:
+    def __init__(self, route_string) -> None:
+        self.prefix = "/" + route_string.split("/")[1]
+        self.route = "/" + route_string.split("/", 2)[2]
+        self.js_route = self.route.replace("{", "${")
+        self.parts = route_string.split("/")[1:]
+        self.var = re.findall(r"\{(.*?)\}", route_string)
+        self.is_function = "{" in self.route
+        self.router_slug = slugify("_".join(self.parts[1:]), separator="_")
+        self.router_camel = camelize(self.router_slug)
+
+
+class RequestType(str, Enum):
+    get = "get"
+    put = "put"
+    post = "post"
+    patch = "patch"
+    delete = "delete"
+
+
+class ParameterIn(str, Enum):
+    query = "query"
+    path = "path"
+
+
+class RouterParameter(BaseModel):
+    required: bool = False
+    name: str
+    location: ParameterIn = Field(..., alias="in")
+
+
+class RequestBody(BaseModel):
+    required: bool = False
+
+
+class HTTPRequest(BaseModel):
+    request_type: RequestType
+    description: str = ""
+    summary: str
+    requestBody: Optional[RequestBody]
+
+    parameters: list[RouterParameter] = []
+    tags: list[str]
+
+    def list_as_js_object_string(self, parameters, braces=True):
+        if len(parameters) == 0:
+            return ""
+
+        if braces:
+            return "{" + ", ".join(parameters) + "}"
+        else:
+            return ", ".join(parameters)
+
+    def payload(self):
+        return "payload" if self.requestBody else ""
+
+    def function_args(self):
+        all_params = [p.name for p in self.parameters]
+        if self.requestBody:
+            all_params.append("payload")
+        return self.list_as_js_object_string(all_params)
+
+    def query_params(self):
+        params = [param.name for param in self.parameters if param.location == ParameterIn.query]
+        return self.list_as_js_object_string(params)
+
+    def path_params(self):
+        params = [param.name for param in self.parameters if param.location == ParameterIn.path]
+        return self.list_as_js_object_string(parameters=params, braces=False)
+
+    @property
+    def summary_camel(self):
+        return camelize(slugify(self.summary))
+
+    @property
+    def js_docs(self):
+        return self.description.replace("\n", "  \n  * ")

dev/code-generation/_static.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+
+CWD = Path(__file__).parent
+
+
+class Directories:
+    out_dir = CWD / "generated"
+
+
+class CodeTemplates:
+    interface = CWD / "templates" / "interface.js"
+    pytest_routes = CWD / "templates" / "test_routes.py.j2"
+
+
+class CodeDest:
+    interface = CWD / "generated" / "interface.js"
+    pytest_routes = CWD / "generated" / "test_routes.py"
+
+
+class CodeKeys:
+    """ Hard coded comment IDs that are used to generate code"""
+
+    nuxt_local_messages = "MESSAGE_LOCALES"
+    nuxt_local_dates = "DATE_LOCALES"

dev/code-generation/gen_nuxt_locales.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+from _gen_utils import inject_inline
+from _static import CodeKeys
+
+PROJECT_DIR = Path(__file__).parent.parent.parent
+
+
+datetime_dir = PROJECT_DIR / "frontend" / "lang" / "dateTimeFormats"
+locales_dir = PROJECT_DIR / "frontend" / "lang" / "messages"
+nuxt_config = PROJECT_DIR / "frontend" / "nuxt.config.js"
+
+"""
+This snippet walks the message and dat locales directories and generates the import information
+for the nuxt.config.js file and automatically injects it into the nuxt.config.js file. Note that
+the code generation ID is hardcoded into the script and required in the nuxt config.
+"""
+
+
+def main():  # sourcery skip: list-comprehension
+    print("Starting...")
+
+    all_date_locales = []
+    for match in datetime_dir.glob("*.json"):
+        all_date_locales.append(f'"{match.stem}": require("./lang/dateTimeFormats/{match.name}"),')
+
+    all_langs = []
+    for match in locales_dir.glob("*.json"):
+        lang_string = f'{{ code: "{match.stem}", file: "{match.name}" }},'
+        all_langs.append(lang_string)
+
+    inject_inline(nuxt_config, CodeKeys.nuxt_local_messages, all_langs)
+    inject_inline(nuxt_config, CodeKeys.nuxt_local_dates, all_date_locales)
+
+    print("Finished...")
+
+
+if __name__ == "__main__":
+    main()

dev/code-generation/gen_pytest_routes.py

@@ -0,0 +1,53 @@
+import json
+from typing import Any
+
+from _gen_utils import render_python_template
+from _open_api_parser import OpenAPIParser
+from _static import CodeDest, CodeTemplates
+from rich.console import Console
+
+from mealie.app import app
+
+"""
+This code is used for generating route objects for each route in the OpenAPI Specification.
+Currently, they are NOT automatically injected into the test suite. As such, you'll need to copy
+the relavent contents of the generated file into the test suite where applicable. I am slowly
+migrating the test suite to use this new generated file and this process will be "automated" in the
+future.
+"""
+
+console = Console()
+
+
+def write_dict_to_file(file_name: str, data: dict[str, Any]):
+    with open(file_name, "w") as f:
+        f.write(json.dumps(data, indent=4))
+
+
+def main():
+    print("Starting...")
+    open_api = OpenAPIParser(app)
+    modules = open_api.get_by_module()
+
+    mods = []
+
+    for mod, value in modules.items():
+
+        routes = []
+        existings = set()
+        # Reduce routes by unique py_route attribute
+        for route in value:
+            if route.py_route not in existings:
+                existings.add(route.py_route)
+                routes.append(route)
+
+        module = {"name": mod, "routes": routes}
+        mods.append(module)
+
+    render_python_template(CodeTemplates.pytest_routes, CodeDest.pytest_routes, {"mods": mods})
+
+    print("Finished...")
+
+
+if __name__ == "__main__":
+    main()

dev/code-generation/templates/test_routes.py.j2

@@ -0,0 +1,9 @@
+{% for mod in mods  %}
+class {{mod.name}}Routes:{% for route in mod.routes  %}{% if not route.path_is_func   %}
+    {{route.name_snake}} = "{{ route.py_route }}"{% endif %}{% endfor %}{% for route in mod.routes  %}
+{% if route.path_is_func   %}
+    @staticmethod
+    def {{route.name_snake}}({{ route.path_vars|join(", ") }}):
+        return f"{{route.py_route}}"       
+{% endif %}{% endfor %}
+{% endfor %}