From 7055cb2c4380560daa5690c17006eb3cace078dc Mon Sep 17 00:00:00 2001 From: Hayden Date: Sat, 23 Oct 2021 16:42:59 -0800 Subject: [PATCH] feat(backend): :sparkles: basic support for background tasks --- mealie/app.py | 2 +- mealie/db/data_access_layer/_access_model.py | 25 ++++++++- .../data_access_layer/access_model_factory.py | 30 ++++++---- mealie/db/models/_all_models.py | 1 + mealie/db/models/_model_base.py | 4 +- mealie/db/models/group/group.py | 2 + mealie/db/models/server/__init__.py | 1 + mealie/db/models/server/task.py | 20 +++++++ mealie/routes/admin/__init__.py | 3 +- mealie/routes/admin/admin_server_tasks.py | 18 ++++++ mealie/schema/server/__init__.py | 1 + mealie/schema/server/tasks.py | 44 +++++++++++++++ mealie/services/server_tasks/__init__.py | 2 + .../server_tasks/background_executory.py | 56 +++++++++++++++++++ .../server_tasks/tasks_http_service.py | 36 ++++++++++++ 15 files changed, 226 insertions(+), 19 deletions(-) create mode 100644 mealie/db/models/server/__init__.py create mode 100644 mealie/db/models/server/task.py create mode 100644 mealie/routes/admin/admin_server_tasks.py create mode 100644 mealie/schema/server/__init__.py create mode 100644 mealie/schema/server/tasks.py create mode 100644 mealie/services/server_tasks/__init__.py create mode 100644 mealie/services/server_tasks/background_executory.py create mode 100644 mealie/services/server_tasks/tasks_http_service.py diff --git a/mealie/app.py b/mealie/app.py index 638c7de8f..0cf880861 100644 --- a/mealie/app.py +++ b/mealie/app.py @@ -93,7 +93,7 @@ def main(): reload_dirs=["mealie"], reload_delay=2, debug=True, - log_level="debug", + log_level="info", use_colors=True, log_config=None, workers=1, diff --git a/mealie/db/data_access_layer/_access_model.py b/mealie/db/data_access_layer/_access_model.py index c10910878..b376689a4 100644 --- a/mealie/db/data_access_layer/_access_model.py +++ b/mealie/db/data_access_layer/_access_model.py @@ -41,22 +41,41 @@ class AccessModel(Generic[T, D]): def get_all(self, limit: int = None, order_by: str = None, start=0, override_schema=None) -> list[T]: eff_schema = override_schema or self.schema + order_attr = None if order_by: order_attr = getattr(self.sql_model, str(order_by)) + order_attr = order_attr.desc() return [ eff_schema.from_orm(x) - for x in self.session.query(self.sql_model).order_by(order_attr.desc()).offset(start).limit(limit).all() + for x in self.session.query(self.sql_model).order_by(order_attr).offset(start).limit(limit).all() ] return [eff_schema.from_orm(x) for x in self.session.query(self.sql_model).offset(start).limit(limit).all()] - def multi_query(self, query_by: dict[str, str], start=0, limit: int = None, override_schema=None) -> list[T]: + def multi_query( + self, + query_by: dict[str, str], + start=0, + limit: int = None, + override_schema=None, + order_by: str = None, + ) -> list[T]: eff_schema = override_schema or self.schema + order_attr = None + if order_by: + order_attr = getattr(self.sql_model, str(order_by)) + order_attr = order_attr.desc() + return [ eff_schema.from_orm(x) - for x in self.session.query(self.sql_model).filter_by(**query_by).offset(start).limit(limit).all() + for x in self.session.query(self.sql_model) + .filter_by(**query_by) + .order_by(order_attr) + .offset(start) + .limit(limit) + .all() ] def get_all_limit_columns(self, fields: list[str], limit: int = None) -> list[D]: diff --git a/mealie/db/data_access_layer/access_model_factory.py b/mealie/db/data_access_layer/access_model_factory.py index cce480414..5dabf64b5 100644 --- a/mealie/db/data_access_layer/access_model_factory.py +++ b/mealie/db/data_access_layer/access_model_factory.py @@ -13,6 +13,7 @@ from mealie.db.models.recipe.comment import RecipeComment from mealie.db.models.recipe.ingredient import IngredientFoodModel, IngredientUnitModel from mealie.db.models.recipe.recipe import RecipeModel from mealie.db.models.recipe.tag import Tag +from mealie.db.models.server.task import ServerTaskModel from mealie.db.models.settings import SiteSettings from mealie.db.models.sign_up import SignUp from mealie.db.models.users import LongLiveToken, User @@ -27,6 +28,7 @@ from mealie.schema.group.webhook import ReadWebhook from mealie.schema.meal_plan.new_meal import ReadPlanEntry from mealie.schema.recipe import CommentOut, Recipe, RecipeCategoryResponse, RecipeTagResponse from mealie.schema.recipe.recipe_ingredient import IngredientFood, IngredientUnit +from mealie.schema.server import ServerTask from mealie.schema.user import GroupInDB, LongLiveTokenInDB, PrivateUser, SignUpOut from mealie.schema.user.user_passwords import PrivatePasswordResetToken @@ -70,15 +72,15 @@ class Database: return RecipeDataAccessModel(self.session, pk_slug, RecipeModel, Recipe) @cached_property - def ingredient_foods(self) -> AccessModel: + def ingredient_foods(self) -> AccessModel[IngredientFood, IngredientFoodModel]: return AccessModel(self.session, pk_id, IngredientFoodModel, IngredientFood) @cached_property - def ingredient_units(self) -> AccessModel: + def ingredient_units(self) -> AccessModel[IngredientUnit, IngredientUnitModel]: return AccessModel(self.session, pk_id, IngredientUnitModel, IngredientUnit) @cached_property - def comments(self) -> AccessModel: + def comments(self) -> AccessModel[CommentOut, RecipeComment]: return AccessModel(self.session, pk_id, RecipeComment, CommentOut) @cached_property @@ -93,19 +95,19 @@ class Database: # Site Items @cached_property - def settings(self) -> AccessModel: + def settings(self) -> AccessModel[SiteSettingsSchema, SiteSettings]: return AccessModel(self.session, pk_id, SiteSettings, SiteSettingsSchema) @cached_property - def sign_up(self) -> AccessModel: + def sign_up(self) -> AccessModel[SignUpOut, SignUp]: return AccessModel(self.session, pk_id, SignUp, SignUpOut) @cached_property - def event_notifications(self) -> AccessModel: + def event_notifications(self) -> AccessModel[EventNotificationIn, EventNotification]: return AccessModel(self.session, pk_id, EventNotification, EventNotificationIn) @cached_property - def events(self) -> AccessModel: + def events(self) -> AccessModel[EventSchema, Event]: return AccessModel(self.session, pk_id, Event, EventSchema) # ================================================================ @@ -116,7 +118,7 @@ class Database: return UserDataAccessModel(self.session, pk_id, User, PrivateUser) @cached_property - def api_tokens(self) -> AccessModel: + def api_tokens(self) -> AccessModel[LongLiveTokenInDB, LongLiveToken]: return AccessModel(self.session, pk_id, LongLiveToken, LongLiveTokenInDB) @cached_property @@ -126,16 +128,20 @@ class Database: # ================================================================ # Group Items + @cached_property + def server_tasks(self) -> AccessModel[ServerTask, ServerTaskModel]: + return AccessModel(self.session, pk_id, ServerTaskModel, ServerTask) + @cached_property def groups(self) -> GroupDataAccessModel: return GroupDataAccessModel(self.session, pk_id, Group, GroupInDB) @cached_property - def group_invite_tokens(self) -> AccessModel: + def group_invite_tokens(self) -> AccessModel[ReadInviteToken, GroupInviteToken]: return AccessModel(self.session, pk_token, GroupInviteToken, ReadInviteToken) @cached_property - def group_preferences(self) -> AccessModel: + def group_preferences(self) -> AccessModel[ReadGroupPreferences, GroupPreferencesModel]: return AccessModel(self.session, "group_id", GroupPreferencesModel, ReadGroupPreferences) @cached_property @@ -143,9 +149,9 @@ class Database: return MealDataAccessModel(self.session, pk_id, GroupMealPlan, ReadPlanEntry) @cached_property - def cookbooks(self) -> AccessModel: + def cookbooks(self) -> AccessModel[ReadCookBook, CookBook]: return AccessModel(self.session, pk_id, CookBook, ReadCookBook) @cached_property - def webhooks(self) -> AccessModel: + def webhooks(self) -> AccessModel[ReadWebhook, GroupWebhooksModel]: return AccessModel(self.session, pk_id, GroupWebhooksModel, ReadWebhook) diff --git a/mealie/db/models/_all_models.py b/mealie/db/models/_all_models.py index afa2ea5c3..7cb43654f 100644 --- a/mealie/db/models/_all_models.py +++ b/mealie/db/models/_all_models.py @@ -1,6 +1,7 @@ from .event import * from .group import * from .recipe.recipe import * +from .server import * from .settings import * from .sign_up import * from .users import * diff --git a/mealie/db/models/_model_base.py b/mealie/db/models/_model_base.py index f5d082a48..907eb90ca 100644 --- a/mealie/db/models/_model_base.py +++ b/mealie/db/models/_model_base.py @@ -9,8 +9,8 @@ from sqlalchemy.orm.session import Session @as_declarative() class Base: id = Column(Integer, primary_key=True) - created_at = Column(DateTime, default=datetime.now()) - update_at = Column(DateTime, default=datetime.now(), onupdate=datetime.now()) + created_at = Column(DateTime, default=datetime.now) + update_at = Column(DateTime, default=datetime.now, onupdate=datetime.now) class BaseMixins: diff --git a/mealie/db/models/group/group.py b/mealie/db/models/group/group.py index 3aceb685a..fdc52e4cb 100644 --- a/mealie/db/models/group/group.py +++ b/mealie/db/models/group/group.py @@ -4,6 +4,7 @@ from sqlalchemy.orm.session import Session from mealie.core.config import get_app_settings from mealie.db.models.group.invite_tokens import GroupInviteToken +from mealie.db.models.server.task import ServerTaskModel from .._model_base import BaseMixins, SqlAlchemyBase from .._model_utils import auto_init @@ -42,6 +43,7 @@ class Group(SqlAlchemyBase, BaseMixins): ) webhooks = orm.relationship(GroupWebhooksModel, uselist=True, cascade="all, delete-orphan") cookbooks = orm.relationship(CookBook, back_populates="group", single_parent=True) + server_tasks = orm.relationship(ServerTaskModel, back_populates="group", single_parent=True) shopping_lists = orm.relationship("ShoppingList", back_populates="group", single_parent=True) @auto_init({"users", "webhooks", "shopping_lists", "cookbooks", "preferences", "invite_tokens", "mealplans"}) diff --git a/mealie/db/models/server/__init__.py b/mealie/db/models/server/__init__.py new file mode 100644 index 000000000..4d00d3583 --- /dev/null +++ b/mealie/db/models/server/__init__.py @@ -0,0 +1 @@ +from .task import * diff --git a/mealie/db/models/server/task.py b/mealie/db/models/server/task.py new file mode 100644 index 000000000..6685218f3 --- /dev/null +++ b/mealie/db/models/server/task.py @@ -0,0 +1,20 @@ +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, orm + +from mealie.db.models._model_base import BaseMixins, SqlAlchemyBase + +from .._model_utils import auto_init + + +class ServerTaskModel(SqlAlchemyBase, BaseMixins): + __tablename__ = "server_tasks" + name = Column(String, nullable=False) + completed_date = Column(DateTime, nullable=True) + status = Column(String, nullable=False) + log = Column(String, nullable=True) + + group_id = Column(Integer, ForeignKey("groups.id")) + group = orm.relationship("Group", back_populates="server_tasks") + + @auto_init() + def __init__(self, **_) -> None: + pass diff --git a/mealie/routes/admin/__init__.py b/mealie/routes/admin/__init__.py index c71f49a52..9aa78c47e 100644 --- a/mealie/routes/admin/__init__.py +++ b/mealie/routes/admin/__init__.py @@ -2,7 +2,7 @@ from fastapi import APIRouter from mealie.routes.routers import AdminAPIRouter -from . import admin_about, admin_email, admin_group, admin_log +from . import admin_about, admin_email, admin_group, admin_log, admin_server_tasks router = AdminAPIRouter(prefix="/admin") @@ -10,3 +10,4 @@ router.include_router(admin_about.router, tags=["Admin: About"]) router.include_router(admin_log.router, tags=["Admin: Log"]) router.include_router(admin_group.router, tags=["Admin: Group"]) router.include_router(admin_email.router, tags=["Admin: Email"]) +router.include_router(admin_server_tasks.router, tags=["Admin: Server Tasks"]) diff --git a/mealie/routes/admin/admin_server_tasks.py b/mealie/routes/admin/admin_server_tasks.py new file mode 100644 index 000000000..858418115 --- /dev/null +++ b/mealie/routes/admin/admin_server_tasks.py @@ -0,0 +1,18 @@ +from fastapi import Depends + +from mealie.routes.routers import UserAPIRouter +from mealie.schema.server.tasks import ServerTask, ServerTaskNames +from mealie.services.server_tasks import BackgroundExecutor, test_executor_func +from mealie.services.server_tasks.tasks_http_service import AdminServerTasks + +router = UserAPIRouter() + + +@router.get("/server-tasks", response_model=list[ServerTask]) +def get_all_tasks(tasks_service: AdminServerTasks = Depends(AdminServerTasks.private)): + return tasks_service.get_all() + + +@router.post("/server-tasks", response_model=ServerTask) +def create_test_tasks(bg_executor: BackgroundExecutor = Depends(BackgroundExecutor.private)): + return bg_executor.dispatch(ServerTaskNames.default, test_executor_func) diff --git a/mealie/schema/server/__init__.py b/mealie/schema/server/__init__.py new file mode 100644 index 000000000..f5456c32b --- /dev/null +++ b/mealie/schema/server/__init__.py @@ -0,0 +1 @@ +from .tasks import * diff --git a/mealie/schema/server/tasks.py b/mealie/schema/server/tasks.py new file mode 100644 index 000000000..6b617bd27 --- /dev/null +++ b/mealie/schema/server/tasks.py @@ -0,0 +1,44 @@ +import datetime +import enum + +from fastapi_camelcase import CamelModel +from pydantic import Field + + +class ServerTaskNames(str, enum.Enum): + default = "Background Task" + backup_task = "Database Backup" + + +class ServerTaskStatus(str, enum.Enum): + running = "running" + finished = "finished" + failed = "failed" + + +class ServerTaskCreate(CamelModel): + group_id: int + name: ServerTaskNames = ServerTaskNames.default + created_at: datetime.datetime = Field(default_factory=datetime.datetime.now) + status: ServerTaskStatus = ServerTaskStatus.running + log: str = "" + + def set_running(self) -> None: + self.status = ServerTaskStatus.running + + def set_finished(self) -> None: + self.status = ServerTaskStatus.finished + + def set_failed(self) -> None: + self.status = ServerTaskStatus.failed + + def append_log(self, message: str) -> None: + # Prefix with Timestamp and append new line and join to log + self.log += f"{datetime.datetime.now()}: {message}\n" + + +class ServerTask(ServerTaskCreate): + id: int + + class Config: + orm_mode = True diff --git a/mealie/services/server_tasks/__init__.py b/mealie/services/server_tasks/__init__.py new file mode 100644 index 000000000..cc4d30bf7 --- /dev/null +++ b/mealie/services/server_tasks/__init__.py @@ -0,0 +1,2 @@ +from .background_executory import * +from .tasks_http_service import * diff --git a/mealie/services/server_tasks/background_executory.py b/mealie/services/server_tasks/background_executory.py new file mode 100644 index 000000000..e5afd244b --- /dev/null +++ b/mealie/services/server_tasks/background_executory.py @@ -0,0 +1,56 @@ +from random import getrandbits +from time import sleep +from typing import Any, Callable + +from sqlalchemy.orm import Session + +from mealie.db.database import get_database +from mealie.schema.server.tasks import ServerTask, ServerTaskCreate, ServerTaskNames + +from .._base_http_service.http_services import UserHttpService + + +class BackgroundExecutor(UserHttpService): + def populate_item(self, _: int) -> ServerTask: + pass + + def dispatch(self, task_name: ServerTaskNames, func: Callable, *args: Any, **kwargs: Any) -> ServerTask: + """The dispatch function is a wrapper around the BackgroundTasks class in Starlett. It dirctly calls + the add_task function and your task will be run in the background. This function all passes the id required + to check on the server tasks in the database and provide updates. + + Tasks that are dispachd by the Background executor should be designed to accept this key word argument + and update the item in the database accordingly. + """ + + server_task = ServerTaskCreate(group_id=self.group_id, name=task_name) + server_task = self.db.server_tasks.create(server_task) + + self.background_tasks.add_task(func, *args, **kwargs, task_id=server_task.id, session=self.session) + + return server_task + + +def test_executor_func(task_id: int, session: Session) -> None: + database = get_database(session) + task = database.server_tasks.get_one(task_id) + + task.append_log("test task has started") + task.append_log("test task sleeping for 60 seconds") + + sleep(60) + + task.append_log("test task has finished sleep") + + # Randomly Decide to set to failed or not + + is_fail = bool(getrandbits(1)) + + if is_fail: + task.append_log("test task has failed") + task.set_failed() + else: + task.append_log("test task has succeeded") + task.set_finished() + + database.server_tasks.update(task.id, task) diff --git a/mealie/services/server_tasks/tasks_http_service.py b/mealie/services/server_tasks/tasks_http_service.py new file mode 100644 index 000000000..d1cf55efa --- /dev/null +++ b/mealie/services/server_tasks/tasks_http_service.py @@ -0,0 +1,36 @@ +from functools import cached_property + +from mealie.schema.server import ServerTask +from mealie.services._base_http_service.http_services import AdminHttpService, UserHttpService + + +class ServerTasksHttpService(UserHttpService[int, ServerTask]): + _restrict_by_group = True + _schema = ServerTask + + @cached_property + def dal(self): + return self.db.server_tasks + + def populate_item(self, id: int) -> ServerTask: + self.item = self.dal.get_one(id) + return self.item + + def get_all(self) -> list[ServerTask]: + return self.dal.multi_query(query_by={"group_id": self.group_id}, order_by="created_at") + + +class AdminServerTasks(AdminHttpService[int, ServerTask]): + _restrict_by_group = True + _schema = ServerTask + + @cached_property + def dal(self): + return self.db.server_tasks + + def populate_item(self, id: int) -> ServerTask: + self.item = self.dal.get_one(id) + return self.item + + def get_all(self) -> list[ServerTask]: + return self.dal.get_all(order_by="created_at")