chore: file generation cleanup (#1736)

This PR does too many things :( 

1. Major refactoring of the dev/scripts and dev/code-generation folders. 

Primarily this was removing duplicate code and cleaning up some poorly written code snippets as well as making them more idempotent so then can be re-run over and over again but still maintain the same results. This is working on my machine, but I've been having problems in CI and comparing diffs so running generators in CI will have to wait. 

2. Re-Implement using the generated api routes for testing

This was a _huge_ refactor that touched damn near every test file but now we have auto-generated typed routes with inline hints and it's used for nearly every test excluding a few that use classes for better parameterization. This should greatly reduce errors when writing new tests. 

3. Minor Perf improvements for the All Recipes endpoint

  A. Removed redundant loops
  B. Uses orjson to do the encoding directly and returns a byte response instead of relying on the default 
       jsonable_encoder.

4. Fix some TS type errors that cropped up for seemingly no reason half way through the PR.

See this issue https://github.com/phillipdupuis/pydantic-to-typescript/issues/28

Basically, the generated TS type is not-correct since Pydantic will automatically fill in null fields. The resulting TS type is generated with a ? to indicate it can be null even though we _know_ that i can't be.

dev/code-generation/_static.py

@@ -1,26 +0,0 @@
-from pathlib import Path
-
-CWD = Path(__file__).parent
-PROJECT_DIR = Path(__file__).parent.parent.parent
-
-
-class Directories:
-    out_dir = CWD / "generated"
-
-
-class CodeTemplates:
-    interface = CWD / "templates" / "interface.js"
-    pytest_routes = CWD / "templates" / "test_routes.py.j2"
-
-
-class CodeDest:
-    interface = CWD / "generated" / "interface.js"
-    pytest_routes = CWD / "generated" / "test_routes.py"
-    use_locales = PROJECT_DIR / "frontend" / "composables" / "use-locales" / "available-locales.ts"
-
-
-class CodeKeys:
-    """Hard coded comment IDs that are used to generate code"""
-
-    nuxt_local_messages = "MESSAGE_LOCALES"
-    nuxt_local_dates = "DATE_LOCALES"

dev/code-generation/gen_docs_api.py

@@ -0,0 +1,46 @@
+import json
+
+from mealie.app import app
+from mealie.core.config import determine_data_dir
+
+DATA_DIR = determine_data_dir()
+
+"""Script to export the ReDoc documentation page into a standalone HTML file."""
+
+HTML_TEMPLATE = """<!-- Custom HTML site displayed as the Home chapter -->
+{% extends "main.html" %}
+{% block tabs %}
+{{ super() }}
+
+<style>
+    body {
+        margin: 0;
+        padding: 0;
+    }
+</style>
+
+
+<div id="redoc-container"></div>
+<script src="https://cdn.jsdelivr.net/npm/redoc/bundles/redoc.standalone.js"> </script>
+<script>
+    var spec = MY_SPECIFIC_TEXT;
+    Redoc.init(spec, {}, document.getElementById("redoc-container"));
+</script>
+
+
+{% endblock %}
+{% block content %}{% endblock %}
+{% block footer %}{% endblock %}
+"""
+
+HTML_PATH = DATA_DIR.parent.parent.joinpath("docs/docs/overrides/api.html")
+
+
+def generate_api_docs(my_app):
+    with open(HTML_PATH, "w") as fd:
+        text = HTML_TEMPLATE.replace("MY_SPECIFIC_TEXT", json.dumps(my_app.openapi()))
+        fd.write(text)
+
+
+if __name__ == "__main__":
+    generate_api_docs(app)

dev/code-generation/gen_py_pytest_data_paths.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from pathlib import Path
 
-from _gen_utils import log, render_python_template
 from slugify import slugify
+from utils import render_python_template
 
 CWD = Path(__file__).parent
 
@@ -25,9 +25,7 @@ class TestDataPath:
 
         # Remove any file extension
         var = var.split(".")[0]
-
         var = var.replace("'", "")
-
         var = slugify(var, separator="_")
 
         return cls(var, rel_path)
@@ -97,8 +95,6 @@ def rename_non_compliant_paths():
 
 
 def main():
-    log.info("Starting Template Generation")
-
     rename_non_compliant_paths()
 
     GENERATED.mkdir(exist_ok=True)
@@ -115,8 +111,6 @@ def main():
         {"children": all_children},
     )
 
-    log.info("Finished Template Generation")
-
 
 if __name__ == "__main__":
     main()

dev/code-generation/gen_py_pytest_routes.py

@@ -0,0 +1,84 @@
+import json
+from pathlib import Path
+
+from fastapi import FastAPI
+from jinja2 import Template
+from pydantic import BaseModel
+from utils import PROJECT_DIR, CodeTemplates, HTTPRequest, RouteObject
+
+CWD = Path(__file__).parent
+
+OUTFILE = PROJECT_DIR / "tests" / "utils" / "api_routes" / "__init__.py"
+
+
+class PathObject(BaseModel):
+    route_object: RouteObject
+    http_verbs: list[HTTPRequest]
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
+def get_path_objects(app: FastAPI):
+    paths = []
+
+    for key, value in app.openapi().items():
+        if key == "paths":
+            for key, value in value.items():
+
+                paths.append(
+                    PathObject(
+                        route_object=RouteObject(key),
+                        http_verbs=[HTTPRequest(request_type=k, **v) for k, v in value.items()],
+                    )
+                )
+
+    return paths
+
+
+def dump_open_api(app: FastAPI):
+    """Writes the Open API as JSON to a json file"""
+    OPEN_API_FILE = CWD / "openapi.json"
+
+    with open(OPEN_API_FILE, "w") as f:
+        f.write(json.dumps(app.openapi()))
+
+
+def read_template(file: Path):
+    with open(file) as f:
+        return f.read()
+
+
+def generate_python_templates(static_paths: list[PathObject], function_paths: list[PathObject]):
+
+    template = Template(read_template(CodeTemplates.pytest_routes))
+    content = template.render(
+        paths={
+            "prefix": "/api",
+            "static_paths": static_paths,
+            "function_paths": function_paths,
+        }
+    )
+    with open(OUTFILE, "w") as f:
+        f.write(content)
+
+    return
+
+
+def main():
+    from mealie.app import app
+
+    dump_open_api(app)
+    paths = get_path_objects(app)
+
+    static_paths = [x.route_object for x in paths if not x.route_object.is_function]
+    function_paths = [x.route_object for x in paths if x.route_object.is_function]
+
+    static_paths.sort(key=lambda x: x.router_slug)
+    function_paths.sort(key=lambda x: x.router_slug)
+
+    generate_python_templates(static_paths, function_paths)
+
+
+if __name__ == "__main__":
+    main()

dev/code-generation/gen_py_schema_exports.py

@@ -0,0 +1,102 @@
+import pathlib
+import re
+from dataclasses import dataclass, field
+
+from utils import PROJECT_DIR, log, render_python_template
+
+template = """# This file is auto-generated by gen_schema_exports.py
+{% for file in data.module.files %}{{ file.import_str() }}
+{% endfor %}
+
+__all__ = [
+    {% for file in data.module.files %}
+    {%- for class in file.classes -%}
+    "{{ class }}",
+    {%- endfor -%}
+    {%- endfor %}
+]
+
+"""
+
+SCHEMA_PATH = PROJECT_DIR / "mealie" / "schema"
+
+SKIP = {"static", "__pycache__"}
+
+
+class PyFile:
+    import_path: str
+    """The import path of the file"""
+
+    classes: list[str]
+    """A list of classes in the file"""
+
+    def __init__(self, path: pathlib.Path):
+        self.import_path = path.stem
+        self.classes = []
+
+        self.classes = PyFile.extract_classes(path)
+        self.classes.sort()
+
+    def import_str(self) -> str:
+        """Returns a string that can be used to import the file"""
+        return f"from .{self.import_path} import {', '.join(self.classes)}"
+
+    @staticmethod
+    def extract_classes(file_path: pathlib.Path) -> list[str]:
+        name = file_path.stem
+
+        if name == "__init__" or name.startswith("_"):
+            return []
+
+        classes = re.findall(r"(?m)^class\s(\w+)", file_path.read_text())
+        return classes
+
+
+@dataclass
+class Modules:
+    directory: pathlib.Path
+    """The directory to search for modules"""
+
+    files: list[PyFile] = field(default_factory=list)
+    """A list of files in the directory"""
+
+    def __post_init__(self):
+        for file in self.directory.glob("*.py"):
+            if file.name.startswith("_"):
+                continue
+
+            pfile = PyFile(file)
+
+            if len(pfile.classes) > 0:
+                self.files.append(pfile)
+
+            else:
+                log.debug(f"Skipping {file.name} as it has no classes")
+
+
+def find_modules(root: pathlib.Path) -> list[Modules]:
+    """Finds all the top level modules in the provided folder"""
+    modules: list[Modules] = []
+    for file in root.iterdir():
+        if file.is_dir() and file.name not in SKIP:
+
+            modules.append(Modules(directory=file))
+
+    return modules
+
+
+def main():
+
+    modules = find_modules(SCHEMA_PATH)
+
+    for module in modules:
+        log.debug(f"Module: {module.directory.name}")
+        for file in module.files:
+            log.debug(f"  File: {file.import_path}")
+            log.debug(f"    Classes: [{', '.join(file.classes)}]")
+
+        render_python_template(template, module.directory / "__init__.py", {"module": module})
+
+
+if __name__ == "__main__":
+    main()

dev/code-generation/gen_pytest_routes.py

@@ -1,53 +0,0 @@
-import json
-from typing import Any
-
-from _gen_utils import render_python_template
-from _open_api_parser import OpenAPIParser
-from _static import CodeDest, CodeTemplates
-from rich.console import Console
-
-from mealie.app import app
-
-"""
-This code is used for generating route objects for each route in the OpenAPI Specification.
-Currently, they are NOT automatically injected into the test suite. As such, you'll need to copy
-the relevant contents of the generated file into the test suite where applicable. I am slowly
-migrating the test suite to use this new generated file and this process will be "automated" in the
-future.
-"""
-
-console = Console()
-
-
-def write_dict_to_file(file_name: str, data: dict[str, Any]):
-    with open(file_name, "w") as f:
-        f.write(json.dumps(data, indent=4))
-
-
-def main():
-    print("Starting...")
-    open_api = OpenAPIParser(app)
-    modules = open_api.get_by_module()
-
-    mods = []
-
-    for mod, value in modules.items():
-
-        routes = []
-        existings = set()
-        # Reduce routes by unique py_route attribute
-        for route in value:
-            if route.py_route not in existings:
-                existings.add(route.py_route)
-                routes.append(route)
-
-        module = {"name": mod, "routes": routes}
-        mods.append(module)
-
-    render_python_template(CodeTemplates.pytest_routes, CodeDest.pytest_routes, {"mods": mods})
-
-    print("Finished...")
-
-
-if __name__ == "__main__":
-    main()

dev/code-generation/gen_schema_exports.py

@@ -1,34 +0,0 @@
-from _gen_utils import log, render_python_template
-from _static import PROJECT_DIR
-
-template = """# GENERATED CODE - DO NOT MODIFY BY HAND
-{% for file in data.files %}from .{{ file }} import *
-{% endfor %}
-"""
-
-SCHEMA_PATH = PROJECT_DIR / "mealie" / "schema"
-
-
-def generate_init_files() -> None:
-    for schema in SCHEMA_PATH.iterdir():
-        if not schema.is_dir():
-            log.info(f"Skipping {schema}")
-            continue
-
-        log.info(f"Generating {schema}")
-        init_file = schema.joinpath("__init__.py")
-
-        module_files = [
-            f.stem for f in schema.iterdir() if f.is_file() and f.suffix == ".py" and not f.stem.startswith("_")
-        ]
-        render_python_template(template, init_file, {"files": module_files})
-
-
-def main():
-    log.info("Starting...")
-    generate_init_files()
-    log.info("Finished...")
-
-
-if __name__ == "__main__":
-    main()

dev/code-generation/gen_ts_locales.py

@@ -1,11 +1,12 @@
 import pathlib
+from pathlib import Path
 
-import _static
 import dotenv
 import requests
-from _gen_utils import log
 from jinja2 import Template
+from pydantic import Extra
 from requests import Response
+from utils import CodeDest, CodeKeys, inject_inline, log
 
 from mealie.schema._mealie import MealieModel
 
@@ -13,9 +14,6 @@ BASE = pathlib.Path(__file__).parent.parent.parent
 
 API_KEY = dotenv.get_key(BASE / ".env", "CROWDIN_API_KEY")
 
-if API_KEY is None or API_KEY == "":
-    log.info("CROWDIN_API_KEY is not set")
-    exit(1)
 
 NAMES = {
     "en-US": "American English",
@@ -71,6 +69,9 @@ class TargetLanguage(MealieModel):
     twoLettersCode: str
     progress: float = 0.0
 
+    class Config:
+        extra = Extra.allow
+
 
 class CrowdinApi:
     project_name = "Mealie"
@@ -127,11 +128,6 @@ class CrowdinApi:
         return response.json()
 
 
-from pathlib import Path
-
-from _gen_utils import inject_inline
-from _static import CodeKeys
-
 PROJECT_DIR = Path(__file__).parent.parent.parent
 
 
@@ -156,7 +152,7 @@ def inject_nuxt_values():
         lang_string = f'{{ code: "{match.stem}", file: "{match.name}" }},'
         all_langs.append(lang_string)
 
-    log.info(f"injecting locales into nuxt config -> {nuxt_config}")
+    log.debug(f"injecting locales into nuxt config -> {nuxt_config}")
     inject_inline(nuxt_config, CodeKeys.nuxt_local_messages, all_langs)
     inject_inline(nuxt_config, CodeKeys.nuxt_local_dates, all_date_locales)
 
@@ -167,18 +163,19 @@ def generate_locales_ts_file():
     tmpl = Template(LOCALE_TEMPLATE)
     rendered = tmpl.render(locales=models)
 
-    log.info(f"generating locales ts file -> {_static.CodeDest.use_locales}")
-    with open(_static.CodeDest.use_locales, "w") as f:
+    log.debug(f"generating locales ts file -> {CodeDest.use_locales}")
+    with open(CodeDest.use_locales, "w") as f:
         f.write(rendered)  # type:ignore
 
 
 def main():
+    if API_KEY is None or API_KEY == "":
+        log.error("CROWDIN_API_KEY is not set")
+        return
+
     generate_locales_ts_file()
-
     inject_nuxt_values()
 
-    log.info("finished code generation")
-
 
 if __name__ == "__main__":
     main()

dev/code-generation/gen_ts_types.py

@@ -1,29 +1,29 @@
 from pathlib import Path
 
-from _gen_utils import log
 from jinja2 import Template
 from pydantic2ts import generate_typescript_defs
+from utils import log
 
 # ============================================================
-# Global Compoenents Generator
 
 template = """// This Code is auto generated by gen_global_components.py
-{% for name in global %} import {{ name }} from "@/components/global/{{ name }}.vue";
+{% for name in global %}import {{ name }} from "@/components/global/{{ name }}.vue";
+{% endfor %}{% for name in layout %}import {{ name }} from "@/components/layout/{{ name }}.vue";
 {% endfor %}
-{% for name in layout %} import {{ name }} from "@/components/layout/{{ name }}.vue";
-{% endfor %}
-
 declare module "vue" {
   export interface GlobalComponents {
     // Global Components
-    {% for name in global %} {{ name }}: typeof {{ name }};
-    {% endfor %} // Layout Components
-    {% for name in layout %} {{ name }}: typeof {{ name }};
-    {% endfor %}
-  }
+{% for name in global -%}
+    {{ "    " }}{{ name }}: typeof {{ name }};
+{% endfor -%}
+    {{ "    " }}// Layout Components
+{% for name in layout -%}
+    {{ "    " }}{{ name }}: typeof {{ name }};
+{% endfor -%}{{ "  }"}}
 }
 
 export {};
+
 """
 
 CWD = Path(__file__).parent
@@ -46,6 +46,7 @@ def generate_global_components_types() -> None:
         data = {}
         for name, path in component_paths.items():
             components = [component.stem for component in path.glob("*.vue")]
+            components.sort()
             data[name] = components
 
         return data
@@ -101,22 +102,27 @@ def generate_typescript_types() -> None:
             failed_modules.append(module)
             log.error(f"Module Error: {e}")
 
-    log.info("\n📁 Skipped Directories:")
+    log.debug("\n📁 Skipped Directories:")
     for skipped_dir in skipped_dirs:
-        log.info(f"   📁 {skipped_dir.name}")
+        log.debug(f"   📁 {skipped_dir.name}")
 
-    log.info("📄 Skipped Files:")
+    log.debug("📄 Skipped Files:")
     for f in skipped_files:
-        log.info(f"   📄 {f.name}")
+        log.debug(f"   📄 {f.name}")
 
-    log.error("❌ Failed Modules:")
-    for f in failed_modules:
-        log.error(f"   ❌ {f.name}")
+    if len(failed_modules) > 0:
+        log.error("❌ Failed Modules:")
+        for f in failed_modules:
+            log.error(f"   ❌ {f.name}")
+
+
+def main():
+    log.debug("\n-- Starting Global Components Generator --")
+    generate_global_components_types()
+
+    log.debug("\n-- Starting Pydantic To Typescript Generator --")
+    generate_typescript_types()
 
 
 if __name__ == "__main__":
-    log.info("\n-- Starting Global Components Generator --")
-    generate_global_components_types()
-
-    log.info("\n-- Starting Pydantic To Typescript Generator --")
-    generate_typescript_types()
+    main()

dev/code-generation/main.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+import gen_py_pytest_data_paths
+import gen_py_pytest_routes
+import gen_py_schema_exports
+import gen_ts_locales
+import gen_ts_types
+from utils import log
+
+CWD = Path(__file__).parent
+
+
+def main():
+    items = [
+        (gen_py_schema_exports.main, "schema exports"),
+        (gen_ts_types.main, "frontend types"),
+        (gen_ts_locales.main, "locales"),
+        (gen_py_pytest_data_paths.main, "test data paths"),
+        (gen_py_pytest_routes.main, "pytest routes"),
+    ]
+
+    for func, name in items:
+        log.info(f"Generating {name}...")
+        func()
+
+
+if __name__ == "__main__":
+    main()

dev/code-generation/templates/test_routes.py.j2

@@ -1,9 +1,11 @@
-{% for mod in mods  %}
-class {{mod.name}}Routes:{% for route in mod.routes  %}{% if not route.path_is_func   %}
-    {{route.name_snake}} = "{{ route.py_route }}"{% endif %}{% endfor %}{% for route in mod.routes  %}
-{% if route.path_is_func   %}
-    @staticmethod
-    def {{route.name_snake}}({{ route.path_vars|join(", ") }}):
-        return f"{{route.py_route}}"       
-{% endif %}{% endfor %}
-{% endfor %}
+# This Content is Auto Generated for Pytest
+prefix = "{{paths.prefix}}"
+{% for path in paths.static_paths %}
+{{ path.router_slug }} = "{{path.prefix}}{{ path.route }}"
+"""`{{path.prefix}}{{ path.route }}`"""{% endfor %}
+{% for path in paths.function_paths  %}
+
+def {{path.router_slug}}({{path.var|join(", ")}}):
+    """`{{ paths.prefix }}{{ path.route }}`"""
+    return f"{prefix}{{ path.route }}"
+{% endfor %}

dev/code-generation/utils/__init__.py

@@ -0,0 +1,25 @@
+from .open_api_parser import OpenAPIParser
+from .route import HTTPRequest, ParameterIn, RequestBody, RequestType, RouteObject, RouterParameter
+from .static import PROJECT_DIR, CodeDest, CodeKeys, CodeTemplates, Directories
+from .template import CodeSlicer, find_start_end, get_indentation_of_string, inject_inline, log, render_python_template
+
+__all__ = [
+    "CodeDest",
+    "CodeKeys",
+    "CodeSlicer",
+    "CodeTemplates",
+    "Directories",
+    "find_start_end",
+    "get_indentation_of_string",
+    "HTTPRequest",
+    "inject_inline",
+    "log",
+    "OpenAPIParser",
+    "ParameterIn",
+    "PROJECT_DIR",
+    "render_python_template",
+    "RequestBody",
+    "RequestType",
+    "RouteObject",
+    "RouterParameter",
+]

dev/code-generation/utils/open_api_parser.py

@@ -3,11 +3,12 @@ import re
 from pathlib import Path
 from typing import Any
 
-from _static import Directories
 from fastapi import FastAPI
 from humps import camelize
 from slugify import slugify
 
+from .static import Directories
+
 
 def get_openapi_spec_by_ref(app, type_reference: str) -> dict:
     if not type_reference:

dev/code-generation/utils/route.py

@@ -3,7 +3,7 @@ from enum import Enum
 from typing import Optional
 
 from humps import camelize
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Extra, Field
 from slugify import slugify
 
 
@@ -30,6 +30,7 @@ class RequestType(str, Enum):
 class ParameterIn(str, Enum):
     query = "query"
     path = "path"
+    header = "header"
 
 
 class RouterParameter(BaseModel):
@@ -37,10 +38,16 @@ class RouterParameter(BaseModel):
     name: str
     location: ParameterIn = Field(..., alias="in")
 
+    class Config:
+        extra = Extra.allow
+
 
 class RequestBody(BaseModel):
     required: bool = False
 
+    class Config:
+        extra = Extra.allow
+
 
 class HTTPRequest(BaseModel):
     request_type: RequestType
@@ -49,7 +56,10 @@ class HTTPRequest(BaseModel):
     requestBody: Optional[RequestBody]
 
     parameters: list[RouterParameter] = []
-    tags: list[str]
+    tags: list[str] | None = []
+
+    class Config:
+        extra = Extra.allow
 
     def list_as_js_object_string(self, parameters, braces=True):
         if len(parameters) == 0:

dev/code-generation/utils/static.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+PARENT = Path(__file__).parent.parent
+PROJECT_DIR = Path(__file__).parent.parent.parent.parent
+
+
+class Directories:
+    out_dir = PARENT / "generated"
+
+
+class CodeTemplates:
+    interface = PARENT / "templates" / "interface.js"
+    pytest_routes = PARENT / "templates" / "test_routes.py.j2"
+
+
+class CodeDest:
+    interface = PARENT / "generated" / "interface.js"
+    pytest_routes = PARENT / "generated" / "test_routes.py"
+    use_locales = PROJECT_DIR / "frontend" / "composables" / "use-locales" / "available-locales.ts"
+
+
+class CodeKeys:
+    """Hard coded comment IDs that are used to generate code"""
+
+    nuxt_local_messages = "MESSAGE_LOCALES"
+    nuxt_local_dates = "DATE_LOCALES"

dev/code-generation/utils/template.py

@@ -22,7 +22,9 @@ def render_python_template(template_file: Path | str, dest: Path, data: dict):
         tplt = Template(template_file)
 
     text = tplt.render(data=data)
+
     text = black.format_str(text, mode=black.FileMode())
+
     dest.write_text(text)
     isort.file(dest)
 
@@ -52,7 +54,7 @@ def get_indentation_of_string(line: str, comment_char: str = "//") -> str:
     return re.sub(rf"{comment_char}.*", "", line).removesuffix("\n")
 
 
-def find_start_end(file_text: list[str], gen_id: str) -> tuple[int, int]:
+def find_start_end(file_text: list[str], gen_id: str) -> tuple[int, int, str]:
     start = None
     end = None
     indentation = None
@@ -90,7 +92,7 @@ def inject_inline(file_path: Path, key: str, code: list[str]) -> None:
 
     """
 
-    with open(file_path, "r") as f:
+    with open(file_path) as f:
         file_text = f.readlines()
 
     start, end, indentation = find_start_end(file_text, key)