chore: file generation cleanup (#1736)

This PR does too many things :( 

1. Major refactoring of the dev/scripts and dev/code-generation folders. 

Primarily this was removing duplicate code and cleaning up some poorly written code snippets as well as making them more idempotent so then can be re-run over and over again but still maintain the same results. This is working on my machine, but I've been having problems in CI and comparing diffs so running generators in CI will have to wait. 

2. Re-Implement using the generated api routes for testing

This was a _huge_ refactor that touched damn near every test file but now we have auto-generated typed routes with inline hints and it's used for nearly every test excluding a few that use classes for better parameterization. This should greatly reduce errors when writing new tests. 

3. Minor Perf improvements for the All Recipes endpoint

  A. Removed redundant loops
  B. Uses orjson to do the encoding directly and returns a byte response instead of relying on the default 
       jsonable_encoder.

4. Fix some TS type errors that cropped up for seemingly no reason half way through the PR.

See this issue https://github.com/phillipdupuis/pydantic-to-typescript/issues/28

Basically, the generated TS type is not-correct since Pydantic will automatically fill in null fields. The resulting TS type is generated with a ? to indicate it can be null even though we _know_ that i can't be.

dev/code-generation/utils/__init__.py

@@ -0,0 +1,25 @@
+from .open_api_parser import OpenAPIParser
+from .route import HTTPRequest, ParameterIn, RequestBody, RequestType, RouteObject, RouterParameter
+from .static import PROJECT_DIR, CodeDest, CodeKeys, CodeTemplates, Directories
+from .template import CodeSlicer, find_start_end, get_indentation_of_string, inject_inline, log, render_python_template
+
+__all__ = [
+    "CodeDest",
+    "CodeKeys",
+    "CodeSlicer",
+    "CodeTemplates",
+    "Directories",
+    "find_start_end",
+    "get_indentation_of_string",
+    "HTTPRequest",
+    "inject_inline",
+    "log",
+    "OpenAPIParser",
+    "ParameterIn",
+    "PROJECT_DIR",
+    "render_python_template",
+    "RequestBody",
+    "RequestType",
+    "RouteObject",
+    "RouterParameter",
+]

dev/code-generation/utils/open_api_parser.py

@@ -0,0 +1,132 @@
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from fastapi import FastAPI
+from humps import camelize
+from slugify import slugify
+
+from .static import Directories
+
+
+def get_openapi_spec_by_ref(app, type_reference: str) -> dict:
+    if not type_reference:
+        return None
+
+    schemas = app["components"]["schemas"]
+    type_text = type_reference.split("/")[-1]
+    return schemas.get(type_text, type_reference)
+
+
+def recursive_dict_search(data: dict[str, Any], key: str) -> Any:
+    """
+    Walks a dictionary searching for a key and returns all the keys
+    matching the provided key"""
+    if key in data:
+        return data[key]
+    for _, v in data.items():
+        if isinstance(v, dict):
+            result = recursive_dict_search(v, key)
+            if result:
+                return result
+    return None
+
+
+class APIFunction:
+    def __init__(self, app, route: str, verb: str, data: dict):
+        self.name_camel = camelize(data.get("summary"))
+        self.name_snake = slugify(data.get("summary"), separator="_")
+
+        self.http_verb = verb
+        self.path_vars = re.findall(r"\{(.*?)\}", route)
+        self.path_is_func = "{" in route
+        self.js_route = route.replace("{", "${")
+        self.py_route = route
+
+        self.body_schema = get_openapi_spec_by_ref(app, recursive_dict_search(data, "$ref"))
+
+    def path_args(self) -> str:
+        return ", ".join(x + ": string | number" for x in self.path_vars)
+
+    # body: Optional[list[str]] = []
+    # path_params: Optional[list[str]] = []
+    # query_params: Optional[list[str]] = []
+
+
+# class APIModule(BaseModel):
+#     name: str
+#     functions: list[APIFunction]
+
+
+class OpenAPIParser:
+    def __init__(self, app: FastAPI) -> None:
+        self.app = app
+        self.spec = app.openapi()
+
+        self.modules = {}
+
+    def dump(self, out_path: Path) -> Path:
+        """Writes the Open API as JSON to a json file"""
+        OPEN_API_FILE = out_path or Directories.out_dir / "openapi.json"
+
+        with open(OPEN_API_FILE, "w") as f:
+            f.write(json.dumps(self.spec, indent=4))
+
+    def _group_by_module_tag(self):
+        """
+        Itterates over all routes and groups them by module. Modules are determined
+        by the suffix text before : in the first tag for the router. These are used
+        to generate the typescript class interface for interacting with the API
+        """
+        modules = {}
+
+        all_paths = self.spec["paths"]
+        for path, http_verbs in all_paths.items():
+            for _, value in http_verbs.items():
+                if "tags" in value:
+                    tag: str = value["tags"][0]
+                    if ":" in tag:
+                        tag = tag.removeprefix('"').split(":")[0].replace(" ", "")
+                        if modules.get(tag):
+                            modules[tag][path] = http_verbs
+                        else:
+                            modules[tag] = {path: http_verbs}
+
+        return modules
+
+    def _get_openapi_spec(self, type_reference: str) -> dict:
+        schemas = self.app["components"]["schemas"]
+        type_text = type_reference.split("/")[-1]
+        return schemas.get(type_text, type_reference)
+
+    def _fill_schema_references(self, raw_modules: dict) -> dict:
+        for _, routes in raw_modules.items():
+            for _, verbs in routes.items():
+                for _, value in verbs.items():
+                    if "requestBody" in value:
+                        try:
+                            schema_ref = value["requestBody"]["content"]["application/json"]["schema"]["$ref"]
+                            schema = self._get_openapi_spec(schema_ref)
+                            value["requestBody"]["content"]["application/json"]["schema"] = schema
+                        except Exception:
+                            continue
+
+        return raw_modules
+
+    def get_by_module(self) -> dict:
+        """Returns paths where tags are split by : and left right is considered the module"""
+        raw_modules = self._group_by_module_tag()
+
+        modules = {}
+        for module_name, routes in raw_modules.items():
+            for route, verbs in routes.items():
+                for verb, value in verbs.items():
+                    function = APIFunction(self.spec, route, verb, value)
+
+                    if modules.get(module_name):
+                        modules[module_name].append(function)
+                    else:
+                        modules[module_name] = [function]
+
+        return modules

dev/code-generation/utils/route.py

@@ -0,0 +1,96 @@
+import re
+from enum import Enum
+from typing import Optional
+
+from humps import camelize
+from pydantic import BaseModel, Extra, Field
+from slugify import slugify
+
+
+class RouteObject:
+    def __init__(self, route_string) -> None:
+        self.prefix = "/" + route_string.split("/")[1]
+        self.route = "/" + route_string.split("/", 2)[2]
+        self.js_route = self.route.replace("{", "${")
+        self.parts = route_string.split("/")[1:]
+        self.var = re.findall(r"\{(.*?)\}", route_string)
+        self.is_function = "{" in self.route
+        self.router_slug = slugify("_".join(self.parts[1:]), separator="_")
+        self.router_camel = camelize(self.router_slug)
+
+
+class RequestType(str, Enum):
+    get = "get"
+    put = "put"
+    post = "post"
+    patch = "patch"
+    delete = "delete"
+
+
+class ParameterIn(str, Enum):
+    query = "query"
+    path = "path"
+    header = "header"
+
+
+class RouterParameter(BaseModel):
+    required: bool = False
+    name: str
+    location: ParameterIn = Field(..., alias="in")
+
+    class Config:
+        extra = Extra.allow
+
+
+class RequestBody(BaseModel):
+    required: bool = False
+
+    class Config:
+        extra = Extra.allow
+
+
+class HTTPRequest(BaseModel):
+    request_type: RequestType
+    description: str = ""
+    summary: str
+    requestBody: Optional[RequestBody]
+
+    parameters: list[RouterParameter] = []
+    tags: list[str] | None = []
+
+    class Config:
+        extra = Extra.allow
+
+    def list_as_js_object_string(self, parameters, braces=True):
+        if len(parameters) == 0:
+            return ""
+
+        if braces:
+            return "{" + ", ".join(parameters) + "}"
+        else:
+            return ", ".join(parameters)
+
+    def payload(self):
+        return "payload" if self.requestBody else ""
+
+    def function_args(self):
+        all_params = [p.name for p in self.parameters]
+        if self.requestBody:
+            all_params.append("payload")
+        return self.list_as_js_object_string(all_params)
+
+    def query_params(self):
+        params = [param.name for param in self.parameters if param.location == ParameterIn.query]
+        return self.list_as_js_object_string(params)
+
+    def path_params(self):
+        params = [param.name for param in self.parameters if param.location == ParameterIn.path]
+        return self.list_as_js_object_string(parameters=params, braces=False)
+
+    @property
+    def summary_camel(self):
+        return camelize(slugify(self.summary))
+
+    @property
+    def js_docs(self):
+        return self.description.replace("\n", "  \n  * ")

dev/code-generation/utils/static.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+PARENT = Path(__file__).parent.parent
+PROJECT_DIR = Path(__file__).parent.parent.parent.parent
+
+
+class Directories:
+    out_dir = PARENT / "generated"
+
+
+class CodeTemplates:
+    interface = PARENT / "templates" / "interface.js"
+    pytest_routes = PARENT / "templates" / "test_routes.py.j2"
+
+
+class CodeDest:
+    interface = PARENT / "generated" / "interface.js"
+    pytest_routes = PARENT / "generated" / "test_routes.py"
+    use_locales = PROJECT_DIR / "frontend" / "composables" / "use-locales" / "available-locales.ts"
+
+
+class CodeKeys:
+    """Hard coded comment IDs that are used to generate code"""
+
+    nuxt_local_messages = "MESSAGE_LOCALES"
+    nuxt_local_dates = "DATE_LOCALES"

dev/code-generation/utils/template.py

@@ -0,0 +1,108 @@
+import logging
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+import black
+import isort
+from jinja2 import Template
+from rich.logging import RichHandler
+
+FORMAT = "%(message)s"
+logging.basicConfig(level=logging.INFO, format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])
+
+log = logging.getLogger("rich")
+
+
+def render_python_template(template_file: Path | str, dest: Path, data: dict):
+    """Render and Format a Jinja2 Template for Python Code"""
+    if isinstance(template_file, Path):
+        tplt = Template(template_file.read_text())
+    else:
+        tplt = Template(template_file)
+
+    text = tplt.render(data=data)
+
+    text = black.format_str(text, mode=black.FileMode())
+
+    dest.write_text(text)
+    isort.file(dest)
+
+
+@dataclass
+class CodeSlicer:
+    start: int
+    end: int
+
+    indentation: str
+    text: list[str]
+
+    _next_line = None
+
+    def purge_lines(self) -> None:
+        start = self.start + 1
+        end = self.end
+        del self.text[start:end]
+
+    def push_line(self, string: str) -> None:
+        self._next_line = self._next_line or self.start + 1
+        self.text.insert(self._next_line, self.indentation + string + "\n")
+        self._next_line += 1
+
+
+def get_indentation_of_string(line: str, comment_char: str = "//") -> str:
+    return re.sub(rf"{comment_char}.*", "", line).removesuffix("\n")
+
+
+def find_start_end(file_text: list[str], gen_id: str) -> tuple[int, int, str]:
+    start = None
+    end = None
+    indentation = None
+
+    for i, line in enumerate(file_text):
+        if "CODE_GEN_ID:" in line and gen_id in line:
+            start = i
+            indentation = get_indentation_of_string(line)
+        if f"END: {gen_id}" in line:
+            end = i
+
+    if start is None or end is None:
+        raise Exception("Could not find start and end of code generation block")
+
+    if start > end:
+        raise Exception(f"Start ({start=}) of code generation block is after end ({end=})")
+
+    return start, end, indentation
+
+
+def inject_inline(file_path: Path, key: str, code: list[str]) -> None:
+    """Injects a list of strings into the file where the key is found in the format defined
+    by the code-generation. Strings are properly indented and a '\n' is added to the end of
+    each string.
+
+    Start -> 'CODE_GEN_ID: <key>'
+    End -> 'END: <key>'
+
+    If no 'CODE_GEN_ID: <key>' is found, and exception is raised
+
+    Args:
+        file_path (Path): Write to file
+        key (str): CODE_GEN_ID: <key>
+        code (list[str]): List of strings to inject.
+
+    """
+
+    with open(file_path) as f:
+        file_text = f.readlines()
+
+    start, end, indentation = find_start_end(file_text, key)
+
+    slicer = CodeSlicer(start, end, indentation, file_text)
+
+    slicer.purge_lines()
+
+    for line in code:
+        slicer.push_line(line)
+
+    with open(file_path, "w") as file:
+        file.writelines(slicer.text)