feat: re-write get all routes to use pagination (#1424)

rewrite get_all routes to use a pagination pattern to allow for better implementations of search, filter, and sorting on the frontend or by any client without fetching all the data. Additionally we added a CI check for running the Nuxt built to confirm that no TS errors were present. Finally, I had to remove the header support for the Shopping lists as the browser caching based off last_updated header was not allowing it to read recent updates due to how we're handling the updated_at property in the database with nested fields. This will have to be looked at in the future to reimplement. I'm unsure how many other routes have a similar issue. 

Co-authored-by: Hayden <64056131+hay-kot@users.noreply.github.com>

mealie/schema/cookbook/cookbook.py

@@ -3,6 +3,7 @@ from slugify import slugify
 
 from mealie.schema._mealie import MealieModel
 from mealie.schema.recipe.recipe import RecipeSummary, RecipeTool
+from mealie.schema.response.pagination import PaginationBase
 
 from ..recipe.recipe_category import CategoryBase, TagBase
 
@@ -51,6 +52,10 @@ class ReadCookBook(UpdateCookBook):
         orm_mode = True
 
 
+class CookBookPagination(PaginationBase):
+    items: list[ReadCookBook]
+
+
 class RecipeCookBook(ReadCookBook):
     group_id: UUID4
     recipes: list[RecipeSummary]

mealie/schema/group/group_events.py

@@ -1,6 +1,7 @@
 from pydantic import UUID4, NoneStr
 
 from mealie.schema._mealie import MealieModel
+from mealie.schema.response.pagination import PaginationBase
 
 # =============================================================================
 # Group Events Notifier Options
@@ -83,6 +84,10 @@ class GroupEventNotifierOut(MealieModel):
         orm_mode = True
 
 
+class GroupEventPagination(PaginationBase):
+    items: list[GroupEventNotifierOut]
+
+
 class GroupEventNotifierPrivate(GroupEventNotifierOut):
     apprise_url: str
 

mealie/schema/group/group_shopping_list.py

@@ -7,6 +7,7 @@ from pydantic import UUID4
 
 from mealie.schema._mealie import MealieModel
 from mealie.schema.recipe.recipe_ingredient import IngredientFood, IngredientUnit
+from mealie.schema.response.pagination import PaginationBase
 
 
 class ShoppingListItemRecipeRef(MealieModel):
@@ -84,6 +85,10 @@ class ShoppingListSummary(ShoppingListSave):
         orm_mode = True
 
 
+class ShoppingListPagination(PaginationBase):
+    items: list[ShoppingListSummary]
+
+
 class ShoppingListUpdate(ShoppingListSummary):
     list_items: list[ShoppingListItemOut] = []
 

mealie/schema/group/webhook.py

@@ -7,6 +7,7 @@ from pydantic import UUID4, validator
 from pydantic.datetime_parse import parse_datetime
 
 from mealie.schema._mealie import MealieModel
+from mealie.schema.response.pagination import PaginationBase
 
 
 class WebhookType(str, enum.Enum):
@@ -57,3 +58,7 @@ class ReadWebhook(SaveWebhook):
 
     class Config:
         orm_mode = True
+
+
+class WebhookPagination(PaginationBase):
+    items: list[ReadWebhook]

mealie/schema/labels/multi_purpose_label.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from pydantic import UUID4
 
 from mealie.schema._mealie import MealieModel
+from mealie.schema.response.pagination import PaginationBase
 
 
 class MultiPurposeLabelCreate(MealieModel):
@@ -25,6 +26,10 @@ class MultiPurposeLabelSummary(MultiPurposeLabelUpdate):
         orm_mode = True
 
 
+class MultiPurposeLabelPagination(PaginationBase):
+    items: list[MultiPurposeLabelSummary]
+
+
 class MultiPurposeLabelOut(MultiPurposeLabelUpdate):
     # shopping_list_items: list[ShoppingListItemOut] = []
     # foods: list[IngredientFood] = []

mealie/schema/meal_plan/plan_rules.py

@@ -4,6 +4,7 @@ from enum import Enum
 from pydantic import UUID4
 
 from mealie.schema._mealie import MealieModel
+from mealie.schema.response.pagination import PaginationBase
 
 
 class Category(MealieModel):
@@ -63,3 +64,7 @@ class PlanRulesOut(PlanRulesSave):
 
     class Config:
         orm_mode = True
+
+
+class PlanRulesPagination(PaginationBase):
+    items: list[PlanRulesOut]

mealie/schema/query.py

@@ -1,10 +0,0 @@
-from typing import Optional
-
-from mealie.schema._mealie import MealieModel
-
-
-class GetAll(MealieModel):
-    start: int = 0
-    limit: int = 999
-    order_by: Optional[str]
-    order_descending: Optional[bool] = True

mealie/schema/recipe/recipe.py

@@ -12,6 +12,7 @@ from slugify import slugify
 from mealie.core.config import get_app_dirs
 from mealie.db.models.recipe.recipe import RecipeModel
 from mealie.schema._mealie import MealieModel
+from mealie.schema.response.pagination import PaginationBase, PaginationQuery
 
 from .recipe_asset import RecipeAsset
 from .recipe_comments import RecipeCommentOut
@@ -32,15 +33,27 @@ class RecipeTag(MealieModel):
         orm_mode = True
 
 
+class RecipeTagPagination(PaginationBase):
+    items: list[RecipeTag]
+
+
 class RecipeCategory(RecipeTag):
     pass
 
 
+class RecipeCategoryPagination(PaginationBase):
+    items: list[RecipeCategory]
+
+
 class RecipeTool(RecipeTag):
     id: UUID4
     on_hand: bool = False
 
 
+class RecipeToolPagination(PaginationBase):
+    items: list[RecipeTool]
+
+
 class CreateRecipeBulk(BaseModel):
     url: str
     categories: list[RecipeCategory] = None
@@ -114,6 +127,14 @@ class RecipeSummary(MealieModel):
         return user_id
 
 
+class RecipePaginationQuery(PaginationQuery):
+    load_food: bool = False
+
+
+class RecipePagination(PaginationBase):
+    items: list[RecipeSummary]
+
+
 class Recipe(RecipeSummary):
     recipe_ingredient: list[RecipeIngredient] = []
     recipe_instructions: Optional[list[RecipeStep]] = []

mealie/schema/recipe/recipe_comments.py

@@ -4,6 +4,7 @@ from typing import Optional
 from pydantic import UUID4
 
 from mealie.schema._mealie import MealieModel
+from mealie.schema.response.pagination import PaginationBase
 
 
 class UserBase(MealieModel):
@@ -39,3 +40,7 @@ class RecipeCommentOut(RecipeCommentCreate):
 
     class Config:
         orm_mode = True
+
+
+class RecipeCommentPagination(PaginationBase):
+    items: list[RecipeCommentOut]

mealie/schema/recipe/recipe_ingredient.py

@@ -8,6 +8,7 @@ from pydantic import UUID4, Field, validator
 
 from mealie.schema._mealie import MealieModel
 from mealie.schema._mealie.types import NoneFloat
+from mealie.schema.response.pagination import PaginationBase
 
 
 class UnitFoodBase(MealieModel):
@@ -31,6 +32,10 @@ class IngredientFood(CreateIngredientFood):
         orm_mode = True
 
 
+class IngredientFoodPagination(PaginationBase):
+    items: list[IngredientFood]
+
+
 class CreateIngredientUnit(UnitFoodBase):
     fraction: bool = True
     abbreviation: str = ""
@@ -48,6 +53,10 @@ class IngredientUnit(CreateIngredientUnit):
         orm_mode = True
 
 
+class IngredientUnitPagination(PaginationBase):
+    items: list[IngredientUnit]
+
+
 class RecipeIngredient(MealieModel):
     title: Optional[str]
     note: Optional[str]

mealie/schema/response/pagination.py

@@ -1,9 +1,13 @@
 import enum
-from typing import Generic, TypeVar
+from typing import Any, Generic, TypeVar
+from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
 
+from humps import camelize
 from pydantic import BaseModel
 from pydantic.generics import GenericModel
 
+from mealie.schema._mealie import MealieModel
+
 DataT = TypeVar("DataT", bound=BaseModel)
 
 
@@ -12,11 +16,11 @@ class OrderDirection(str, enum.Enum):
     desc = "desc"
 
 
-class PaginationQuery(BaseModel):
+class PaginationQuery(MealieModel):
     page: int = 1
+    per_page: int = 50
     order_by: str = "created_at"
     order_direction: OrderDirection = OrderDirection.desc
-    per_page: int = 50
 
 
 class PaginationBase(GenericModel, Generic[DataT]):
@@ -24,4 +28,45 @@ class PaginationBase(GenericModel, Generic[DataT]):
     per_page: int = 10
     total: int = 0
     total_pages: int = 0
-    data: list[DataT]
+    items: list[DataT]
+    next: str | None
+    previous: str | None
+
+    def _set_next(self, route: str, query_params: dict[str, Any]) -> None:
+        if self.page >= self.total_pages:
+            self.next = None
+            return
+
+        # combine params with base route
+        query_params["page"] = self.page + 1
+        self.next = PaginationBase.merge_query_parameters(route, query_params)
+
+    def _set_prev(self, route: str, query_params: dict[str, Any]) -> None:
+        if self.page <= 1:
+            self.previous = None
+            return
+
+        # combine params with base route
+        query_params["page"] = self.page - 1
+        self.previous = PaginationBase.merge_query_parameters(route, query_params)
+
+    def set_pagination_guides(self, route: str, query_params: dict[str, Any] | None) -> None:
+        if not query_params:
+            query_params = {}
+
+        query_params = camelize(query_params)
+
+        # sanitize user input
+        self.page = max(self.page, 1)
+        self._set_next(route, query_params)
+        self._set_prev(route, query_params)
+
+    @staticmethod
+    def merge_query_parameters(url: str, params: dict[str, Any]):
+        scheme, netloc, path, query_string, fragment = urlsplit(url)
+
+        query_params = parse_qs(query_string)
+        query_params.update(params)
+        new_query_string = urlencode(query_params, doseq=True)
+
+        return urlunsplit((scheme, netloc, path, new_query_string, fragment))

mealie/schema/server/tasks.py

@@ -5,6 +5,7 @@ from uuid import UUID
 from pydantic import Field
 
 from mealie.schema._mealie import MealieModel
+from mealie.schema.response.pagination import PaginationBase
 
 
 class ServerTaskNames(str, enum.Enum):
@@ -45,3 +46,7 @@ class ServerTask(ServerTaskCreate):
 
     class Config:
         orm_mode = True
+
+
+class ServerTaskPagination(PaginationBase):
+    items: list[ServerTask]

mealie/schema/user/user.py

@@ -11,6 +11,7 @@ from mealie.db.models.users import User
 from mealie.schema._mealie import MealieModel
 from mealie.schema.group.group_preferences import ReadGroupPreferences
 from mealie.schema.recipe import RecipeSummary
+from mealie.schema.response.pagination import PaginationBase
 
 from ..recipe import CategoryBase
 
@@ -113,6 +114,10 @@ class UserOut(UserBase):
             }
 
 
+class UserPagination(PaginationBase):
+    items: list[UserOut]
+
+
 class UserFavorites(UserBase):
     favorite_recipes: list[RecipeSummary] = []  # type: ignore
 
@@ -180,6 +185,10 @@ class GroupInDB(UpdateGroup):
         return GroupInDB.get_export_directory(self.id)
 
 
+class GroupPagination(PaginationBase):
+    items: list[GroupInDB]
+
+
 class LongLiveTokenInDB(CreateToken):
     id: int
     user: PrivateUser