diff --git a/dev/code-generation/gen_py_pytest_routes.py b/dev/code-generation/gen_py_pytest_routes.py index f433abf77..88c54201c 100644 --- a/dev/code-generation/gen_py_pytest_routes.py +++ b/dev/code-generation/gen_py_pytest_routes.py @@ -17,9 +17,16 @@ class PathObject(BaseModel): http_verbs: list[HTTPRequest] -def get_path_objects(app: FastAPI): - paths = [] +def force_include_in_schema(app: FastAPI): + # clear schema cache + app.openapi_schema = None + for route in app.routes: + route.include_in_schema = True + +def get_path_objects(app: FastAPI): + force_include_in_schema(app) + paths = [] for key, value in app.openapi().items(): if key == "paths": for key, value2 in value.items(): diff --git a/docs/docs/documentation/getting-started/installation/backend-config.md b/docs/docs/documentation/getting-started/installation/backend-config.md index 11490e62d..8dd913dc5 100644 --- a/docs/docs/documentation/getting-started/installation/backend-config.md +++ b/docs/docs/documentation/getting-started/installation/backend-config.md @@ -106,14 +106,15 @@ For usage, see [Usage - OpenID Connect](../authentication/oidc.md) Mealie supports various integrations using OpenAI. To enable OpenAI, [you must provide your OpenAI API key](https://platform.openai.com/api-keys). You can tweak how OpenAI is used using these backend settings. Please note that while OpenAI usage is optimized to reduce API costs, you're unlikely to be able to use OpenAI features with the free tier limits. -| Variables | Default | Description | -| ------------------------- | :-----: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| OPENAI_BASE_URL | None | The base URL for the OpenAI API. If you're not sure, leave this empty to use the standard OpenAI platform | -| OPENAI_API_KEY | None | Your OpenAI API Key. Enables OpenAI-related features | -| OPENAI_MODEL | gpt-4o | Which OpenAI model to use. If you're not sure, leave this empty | -| OPENAI_WORKERS | 2 | Number of OpenAI workers per request. Higher values may increase processing speed, but will incur additional API costs | -| OPENAI_SEND_DATABASE_DATA | True | Whether to send Mealie data to OpenAI to improve request accuracy. This will incur additional API costs | -| OPENAI_REQUEST_TIMEOUT | 10 | The number of seconds to wait for an OpenAI request to complete before cancelling the request. Leave this empty unless you're running into timeout issues on slower hardware | +| Variables | Default | Description | +| ---------------------------- | :-----: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| OPENAI_BASE_URL | None | The base URL for the OpenAI API. If you're not sure, leave this empty to use the standard OpenAI platform | +| OPENAI_API_KEY | None | Your OpenAI API Key. Enables OpenAI-related features | +| OPENAI_MODEL | gpt-4o | Which OpenAI model to use. If you're not sure, leave this empty | +| OPENAI_ENABLE_IMAGE_SERVICES | True | Whether to enable OpenAI image services, such as creating recipes via image. Leave this enabled unless your custom model doesn't support it, or you want to reduce costs | +| OPENAI_WORKERS | 2 | Number of OpenAI workers per request. Higher values may increase processing speed, but will incur additional API costs | +| OPENAI_SEND_DATABASE_DATA | True | Whether to send Mealie data to OpenAI to improve request accuracy. This will incur additional API costs | +| OPENAI_REQUEST_TIMEOUT | 60 | The number of seconds to wait for an OpenAI request to complete before cancelling the request. Leave this empty unless you're running into timeout issues on slower hardware | ### Themeing diff --git a/frontend/components/Layout/DefaultLayout.vue b/frontend/components/Layout/DefaultLayout.vue index c7bbd03fc..4c9c47afa 100644 --- a/frontend/components/Layout/DefaultLayout.vue +++ b/frontend/components/Layout/DefaultLayout.vue @@ -1,25 +1,26 @@ + + + {{ $globals.icons.menu }} + + + + + + + + + - + return { + groupSlug, + cookbookLinks, + createLinks, + bottomLinks, + topLinks, + isAdmin, + isOwnGroup, + languageDialog, + toggleDark, + sidebar, + }; + }, +}); + diff --git a/frontend/components/global/ImageCropper.vue b/frontend/components/global/ImageCropper.vue index a4c250e4a..c60279297 100644 --- a/frontend/components/global/ImageCropper.vue +++ b/frontend/components/global/ImageCropper.vue @@ -1,6 +1,16 @@ diff --git a/frontend/lang/messages/en-US.json b/frontend/lang/messages/en-US.json index 62e0f34d0..ae28f3267 100644 --- a/frontend/lang/messages/en-US.json +++ b/frontend/lang/messages/en-US.json @@ -561,7 +561,12 @@ "create-recipe-description": "Create a new recipe from scratch.", "create-recipes": "Create Recipes", "import-with-zip": "Import with .zip", - "create-recipe-from-an-image": "Create recipe from an image", + "create-recipe-from-an-image": "Create Recipe from an Image", + "create-recipe-from-an-image-description": "Create a recipe by uploading an image of it. Mealie will attempt to extract the text from the image using AI and create a recipe from it.", + "crop-and-rotate-the-image": "Crop and rotate the image so that only the text is visible, and it's in the correct orientation.", + "create-from-image": "Create from Image", + "should-translate-description": "Translate the recipe into my language", + "please-wait-image-procesing": "Please wait, the image is processing. This may take some time.", "bulk-url-import": "Bulk URL Import", "debug-scraper": "Debug Scraper", "create-a-recipe-by-providing-the-name-all-recipes-must-have-unique-names": "Create a recipe by providing the name. All recipes must have unique names.", diff --git a/frontend/lib/api/types/admin.ts b/frontend/lib/api/types/admin.ts index 6fdeec6a2..b11f26dda 100644 --- a/frontend/lib/api/types/admin.ts +++ b/frontend/lib/api/types/admin.ts @@ -42,6 +42,7 @@ export interface AppInfo { oidcRedirect: boolean; oidcProviderName: string; enableOpenai: boolean; + enableOpenaiImageServices: boolean; } export interface AppStartupInfo { isFirstLogin: boolean; diff --git a/frontend/lib/api/user/recipes/recipe.ts b/frontend/lib/api/user/recipes/recipe.ts index f899d3b4b..8aaeb3172 100644 --- a/frontend/lib/api/user/recipes/recipe.ts +++ b/frontend/lib/api/user/recipes/recipe.ts @@ -35,6 +35,7 @@ const routes = { recipesCreateUrl: `${prefix}/recipes/create-url`, recipesCreateUrlBulk: `${prefix}/recipes/create-url/bulk`, recipesCreateFromZip: `${prefix}/recipes/create-from-zip`, + recipesCreateFromImage: `${prefix}/recipes/create-from-image`, recipesCategory: `${prefix}/recipes/category`, recipesParseIngredient: `${prefix}/parser/ingredient`, recipesParseIngredients: `${prefix}/parser/ingredients`, @@ -140,6 +141,19 @@ export class RecipeAPI extends BaseCRUDAPI { return await this.requests.post(routes.recipesCreateUrlBulk, payload); } + async createOneFromImage(fileObject: Blob | File, fileName: string, translateLanguage: string | null = null) { + const formData = new FormData(); + formData.append("images", fileObject); + formData.append("extension", fileName.split(".").pop() ?? ""); + + let apiRoute = routes.recipesCreateFromImage + if (translateLanguage) { + apiRoute = `${apiRoute}?translateLanguage=${translateLanguage}` + } + + return await this.requests.post(apiRoute, formData); + } + async parseIngredients(parser: Parser, ingredients: Array) { parser = parser || "nlp"; return await this.requests.post(routes.recipesParseIngredients, { parser, ingredients }); diff --git a/frontend/pages/g/_groupSlug/r/create.vue b/frontend/pages/g/_groupSlug/r/create.vue index 2fd3f4bdf..532149f4f 100644 --- a/frontend/pages/g/_groupSlug/r/create.vue +++ b/frontend/pages/g/_groupSlug/r/create.vue @@ -28,6 +28,7 @@ diff --git a/mealie/core/settings/settings.py b/mealie/core/settings/settings.py index 1b33010ea..3ec9208ca 100644 --- a/mealie/core/settings/settings.py +++ b/mealie/core/settings/settings.py @@ -304,6 +304,8 @@ class AppSettings(AppLoggingSettings): """Your OpenAI API key. Required to enable OpenAI features""" OPENAI_MODEL: str = "gpt-4o" """Which OpenAI model to send requests to. Leave this unset for most usecases""" + OPENAI_ENABLE_IMAGE_SERVICES: bool = True + """Whether to enable image-related features in OpenAI""" OPENAI_WORKERS: int = 2 """ Number of OpenAI workers per request. Higher values may increase @@ -314,8 +316,7 @@ class AppSettings(AppLoggingSettings): Sending database data may increase accuracy in certain requests, but will incur additional API costs """ - - OPENAI_REQUEST_TIMEOUT: int = 10 + OPENAI_REQUEST_TIMEOUT: int = 60 """ The number of seconds to wait for an OpenAI request to complete before cancelling the request """ diff --git a/mealie/routes/admin/admin_about.py b/mealie/routes/admin/admin_about.py index edc69f2ab..ddb7f9574 100644 --- a/mealie/routes/admin/admin_about.py +++ b/mealie/routes/admin/admin_about.py @@ -34,6 +34,7 @@ class AdminAboutController(BaseAdminController): oidc_redirect=settings.OIDC_AUTO_REDIRECT, oidc_provider_name=settings.OIDC_PROVIDER_NAME, enable_openai=settings.OPENAI_ENABLED, + enable_openai_image_services=settings.OPENAI_ENABLED and settings.OPENAI_ENABLE_IMAGE_SERVICES, ) @router.get("/statistics", response_model=AppStatistics) diff --git a/mealie/routes/app/app_about.py b/mealie/routes/app/app_about.py index 6aff77c97..a2d00f196 100644 --- a/mealie/routes/app/app_about.py +++ b/mealie/routes/app/app_about.py @@ -33,6 +33,7 @@ def get_app_info(session: Session = Depends(generate_session)): oidc_redirect=settings.OIDC_AUTO_REDIRECT, oidc_provider_name=settings.OIDC_PROVIDER_NAME, enable_openai=settings.OPENAI_ENABLED, + enable_openai_image_services=settings.OPENAI_ENABLED and settings.OPENAI_ENABLE_IMAGE_SERVICES, ) diff --git a/mealie/routes/recipe/recipe_crud_routes.py b/mealie/routes/recipe/recipe_crud_routes.py index 0864986fe..0d34992ce 100644 --- a/mealie/routes/recipe/recipe_crud_routes.py +++ b/mealie/routes/recipe/recipe_crud_routes.py @@ -254,6 +254,9 @@ class RecipeController(BaseRecipeController): return "recipe_scrapers was unable to scrape this URL" + # ================================================================================================================== + # Other Create Operations + @router.post("/create-from-zip", status_code=201) def create_recipe_from_zip(self, archive: UploadFile = File(...)): """Create recipe from archive""" @@ -266,6 +269,31 @@ class RecipeController(BaseRecipeController): return recipe.slug + @router.post("/create-from-image", status_code=201) + async def create_recipe_from_image( + self, + images: list[UploadFile] = File(...), + translate_language: str | None = Query(None, alias="translateLanguage"), + ): + """ + Create a recipe from an image using OpenAI. + Optionally specify a language for it to translate the recipe to. + """ + + if not (self.settings.OPENAI_ENABLED and self.settings.OPENAI_ENABLE_IMAGE_SERVICES): + raise HTTPException( + status_code=400, + detail=ErrorResponse.respond("OpenAI image services are not enabled"), + ) + + recipe = await self.service.create_from_images(images, translate_language) + self.publish_event( + event_type=EventTypes.recipe_created, + document_data=EventRecipeData(operation=EventOperation.create, recipe_slug=recipe.slug), + ) + + return recipe.slug + # ================================================================================================================== # CRUD Operations diff --git a/mealie/schema/admin/about.py b/mealie/schema/admin/about.py index e6451df9a..a91c7b48f 100644 --- a/mealie/schema/admin/about.py +++ b/mealie/schema/admin/about.py @@ -19,6 +19,7 @@ class AppInfo(MealieModel): oidc_redirect: bool oidc_provider_name: str enable_openai: bool + enable_openai_image_services: bool class AppTheme(MealieModel): diff --git a/mealie/schema/openai/_base.py b/mealie/schema/openai/_base.py index 3a9f665e4..bcf749665 100644 --- a/mealie/schema/openai/_base.py +++ b/mealie/schema/openai/_base.py @@ -1,5 +1,9 @@ from pydantic import BaseModel +from mealie.core.root_logger import get_logger + +logger = get_logger() + class OpenAIBase(BaseModel): """ @@ -8,3 +12,16 @@ class OpenAIBase(BaseModel): """ __doc__ = "" # we don't want to include the docstring in the JSON schema + + @classmethod + def parse_openai_response(cls, response: str | None): + """ + This method should be implemented in the child class. It should + parse the JSON response from OpenAI and return a dictionary. + """ + + try: + return cls.model_validate_json(response or "") + except Exception: + logger.debug(f"Failed to parse OpenAI response as {cls}. Response: {response}") + raise diff --git a/mealie/schema/openai/recipe.py b/mealie/schema/openai/recipe.py new file mode 100644 index 000000000..146345b97 --- /dev/null +++ b/mealie/schema/openai/recipe.py @@ -0,0 +1,205 @@ +from textwrap import dedent + +from pydantic import Field + +from ._base import OpenAIBase + + +class OpenAIRecipeIngredient(OpenAIBase): + title: str | None = Field( + None, + description=dedent( + """ + The title of the section of the recipe that the ingredient is found in. Recipes may not specify + ingredient sections, in which case this should be left blank. + Only the first item in the section should have this set, + whereas subsuquent items should have their titles left blank (unless they start a new section). + """ + ), + ) + + text: str = Field( + ..., + description=dedent( + """ + The text of the ingredient. This should represent the entire ingredient, such as "1 cup of flour" or + "2 cups of onions, chopped". If the ingredient is completely blank, skip it and do not add the ingredient, + since this field is required. + + If the ingredient has no text, but has a title, include the title on the + next ingredient instead. + """ + ), + ) + + +class OpenAIRecipeInstruction(OpenAIBase): + title: str | None = Field( + None, + description=dedent( + """ + The title of the section of the recipe that the instruction is found in. Recipes may not specify + instruction sections, in which case this should be left blank. + Only the first instruction in the section should have this set, + whereas subsuquent instructions should have their titles left blank (unless they start a new section). + """ + ), + ) + + text: str = Field( + ..., + description=dedent( + """ + The text of the instruction. This represents one step in the recipe, such as "Preheat the oven to 350", + or "Sauté the onions for 20 minutes". Sometimes steps can be longer, such as "Bring a large pot of lightly + salted water to a boil. Add ditalini pasta and cook for 8 minutes or until al dente; drain.". + + Sometimes, but not always, recipes will include their number in front of the text, such as + "1.", "2.", or "Step 1", "Step 2", or "First", "Second". In the case where they are directly numbered + ("1.", "2.", "Step one", "Step 1", "Step two", "Step 2", etc.), you should not include the number in + the text. However, if they use words ("First", "Second", etc.), then those should be included. + + If the instruction is completely blank, skip it and do not add the instruction, since this field is + required. If the ingredient has no text, but has a title, include the title on the next + instruction instead. + """ + ), + ) + + +class OpenAIRecipeNotes(OpenAIBase): + title: str | None = Field( + None, + description=dedent( + """ + The title of the note. Notes may not specify a title, and just have a body of text. In this case, + title should be left blank, and all content should go in the note text. If the note title is just + "note" or "info", you should ignore it and leave the title blank. + """ + ), + ) + + text: str = Field( + ..., + description=dedent( + """ + The text of the note. This should represent the entire note, such as "This recipe is great for + a summer picnic" or "This recipe is a family favorite". They may also include additional prep + instructions such as "to make this recipe gluten free, use gluten free flour", or "you may prepare + the dough the night before and refrigerate it until ready to bake". + + If the note is completely blank, skip it and do not add the note, since this field is required. + """ + ), + ) + + +class OpenAIRecipe(OpenAIBase): + name: str = Field( + ..., + description=dedent( + """ + The name or title of the recipe. If you're unable to determine the name of the recipe, you should + make your best guess based upon the ingredients and instructions provided. + """ + ), + ) + + description: str | None = Field( + ..., + description=dedent( + """ + A long description of the recipe. This should be a string that describes the recipe in a few words + or sentences. If the recipe doesn't have a description, you should return None. + """ + ), + ) + + recipe_yield: str | None = Field( + None, + description=dedent( + """ + The yield of the recipe. For instance, if the recipe makes 12 cookies, the yield is "12 cookies". + If the recipe makes 2 servings, the yield is "2 servings". Typically yield consists of a number followed + by the word "serving" or "servings", but it can be any string that describes the yield. If the yield + isn't specified, you should return None. + """ + ), + ) + + total_time: str | None = Field( + None, + description=dedent( + """ + The total time it takes to make the recipe. This should be a string that describes a duration of time, + such as "1 hour and 30 minutes", "90 minutes", or "1.5 hours". If the recipe has multiple times, choose + the longest time. If the recipe doesn't specify a total time or duration, or it specifies a prep time or + perform time but not a total time, you should return None. Do not duplicate times between total time, prep + time and perform time. + """ + ), + ) + + prep_time: str | None = Field( + None, + description=dedent( + """ + The time it takes to prepare the recipe. This should be a string that describes a duration of time, + such as "30 minutes", "1 hour", or "1.5 hours". If the recipe has a total time, the prep time should be + less than the total time. If the recipe doesn't specify a prep time, you should return None. If the recipe + supplies only one time, it should be the total time. Do not duplicate times between total time, prep + time and coperformok time. + """ + ), + ) + + perform_time: str | None = Field( + None, + description=dedent( + """ + The time it takes to cook the recipe. This should be a string that describes a duration of time, + such as "30 minutes", "1 hour", or "1.5 hours". If the recipe has a total time, the perform time should be + less than the total time. If the recipe doesn't specify a perform time, you should return None. If the + recipe specifies a cook time, active time, or other time besides total or prep, you should use that + time as the perform time. If the recipe supplies only one time, it should be the total time, and not the + perform time. Do not duplicate times between total time, prep time and perform time. + """ + ), + ) + + ingredients: list[OpenAIRecipeIngredient] = Field( + [], + description=dedent( + """ + A list of ingredients used in the recipe. Ingredients should be inserted in the order they appear in the + recipe. If the recipe has no ingredients, you should return an empty list. + + Often times, but not always, ingredients are separated by line breaks. Use these as a guide to + separate ingredients. + """ + ), + ) + + instructions: list[OpenAIRecipeInstruction] = Field( + [], + description=dedent( + """ + A list of ingredients used in the recipe. Ingredients should be inserted in the order they appear in the + recipe. If the recipe has no ingredients, you should return an empty list. + + Often times, but not always, instructions are separated by line breaks and/or separated by paragraphs. + Use these as a guide to separate instructions. They also may be separated by numbers or words, such as + "1.", "2.", "Step 1", "Step 2", "First", "Second", etc. + """ + ), + ) + + notes: list[OpenAIRecipeNotes] = Field( + [], + description=dedent( + """ + A list of notes found in the recipe. Notes should be inserted in the order they appear in the recipe. + They may appear anywhere on the recipe, though they are typically found under the instructions. + """ + ), + ) diff --git a/mealie/services/openai/__init__.py b/mealie/services/openai/__init__.py index 8db69e2eb..ae9bc901d 100644 --- a/mealie/services/openai/__init__.py +++ b/mealie/services/openai/__init__.py @@ -1,6 +1,8 @@ -from .openai import OpenAIDataInjection, OpenAIService +from .openai import OpenAIDataInjection, OpenAIImageExternal, OpenAILocalImage, OpenAIService __all__ = [ "OpenAIDataInjection", + "OpenAIImageExternal", + "OpenAILocalImage", "OpenAIService", ] diff --git a/mealie/services/openai/openai.py b/mealie/services/openai/openai.py index 4e31045de..db7f9fc3b 100644 --- a/mealie/services/openai/openai.py +++ b/mealie/services/openai/openai.py @@ -1,6 +1,8 @@ +import base64 import inspect import json import os +from abc import ABC, abstractmethod from pathlib import Path from textwrap import dedent @@ -9,6 +11,7 @@ from openai.resources.chat.completions import ChatCompletion from pydantic import BaseModel, field_validator from mealie.core.config import get_app_settings +from mealie.pkgs import img from .._base_service import BaseService @@ -39,6 +42,37 @@ class OpenAIDataInjection(BaseModel): return value +class OpenAIImageBase(BaseModel, ABC): + @abstractmethod + def get_image_url(self) -> str: ... + + def build_message(self) -> dict: + return { + "type": "image_url", + "image_url": {"url": self.get_image_url()}, + } + + +class OpenAIImageExternal(OpenAIImageBase): + url: str + + def get_image_url(self) -> str: + return self.url + + +class OpenAILocalImage(OpenAIImageBase): + filename: str + path: Path + + def get_image_url(self) -> str: + image = img.PillowMinifier.to_webp( + self.path, dest=self.path.parent.joinpath(f"{self.filename}-min-original.webp") + ) + with open(image, "rb") as f: + b64content = base64.b64encode(f.read()).decode("utf-8") + return f"data:image/webp;base64,{b64content}" + + class OpenAIService(BaseService): PROMPTS_DIR = Path(os.path.dirname(os.path.abspath(__file__))) / "prompts" @@ -50,6 +84,7 @@ class OpenAIService(BaseService): self.model = settings.OPENAI_MODEL self.workers = settings.OPENAI_WORKERS self.send_db_data = settings.OPENAI_SEND_DATABASE_DATA + self.enable_image_services = settings.OPENAI_ENABLE_IMAGE_SERVICES self.get_client = lambda: AsyncOpenAI( base_url=settings.OPENAI_BASE_URL, @@ -99,7 +134,7 @@ class OpenAIService(BaseService): return "\n".join(content_parts) async def _get_raw_response( - self, prompt: str, message: str, temperature=0.2, force_json_response=True + self, prompt: str, content: list[dict], temperature=0.2, force_json_response=True ) -> ChatCompletion: client = self.get_client() return await client.chat.completions.create( @@ -110,7 +145,7 @@ class OpenAIService(BaseService): }, { "role": "user", - "content": message, + "content": content, }, ], model=self.model, @@ -118,10 +153,26 @@ class OpenAIService(BaseService): response_format={"type": "json_object"} if force_json_response else NOT_GIVEN, ) - async def get_response(self, prompt: str, message: str, temperature=0.2, force_json_response=True) -> str | None: + async def get_response( + self, + prompt: str, + message: str, + *, + images: list[OpenAIImageBase] | None = None, + temperature=0.2, + force_json_response=True, + ) -> str | None: """Send data to OpenAI and return the response message content""" + if images and not self.enable_image_services: + self.logger.warning("OpenAI image services are disabled, ignoring images") + images = None + try: - response = await self._get_raw_response(prompt, message, temperature, force_json_response) + user_messages = [{"type": "text", "text": message}] + for image in images or []: + user_messages.append(image.build_message()) + + response = await self._get_raw_response(prompt, user_messages, temperature, force_json_response) if not response.choices: return None return response.choices[0].message.content diff --git a/mealie/services/openai/prompts/recipes/parse-recipe-image.txt b/mealie/services/openai/prompts/recipes/parse-recipe-image.txt new file mode 100644 index 000000000..e40ae9fd3 --- /dev/null +++ b/mealie/services/openai/prompts/recipes/parse-recipe-image.txt @@ -0,0 +1,11 @@ +You are a bot that reads an image, or a set of images, and parses it into recipe JSON. You will receive an image from the user and you need to extract the recipe data and return its JSON in valid schema. The recipe schema will be included at the bottom of this message. + +It is imperative that you do not create any data or otherwise make up any information. Failure to adhere to this rule is illegal and will result in harsh punishment. If you are unable to extract data due to insufficient input, you may reply with a completely empty JSON object (represented by two brackets: {}). + +Do not under any circumstances insert data not found directly in the image. Ingredients, instructions, and notes should come directly from the image and not be generated or otherwise made up. It is illegal for you to create information not found directly in the image. + +Your response must be in valid JSON in the provided Recipe definition below. You must respond in this JSON schema; failure to do so is illegal. It is imperative that you follow the schema precisely to avoid punishment. You must follow the JSON schema. + +The user message that you receive will be one or more images. Assume all images provided belong to a single recipe, not multiple recipes. The recipe may consist of printed text or handwritten text. It may be rotated or not properly cropped. It is your job to figure out which part of the image is the important content and extract it. + +The text you receive in the provided image or images may not be in English. The user may provide a language for you to translate the recipe into. If the user doesn't ask for a translation, you should preserve the text as-is without translating or otherwise modifying it. Otherwise, you should translate all text (recipe name, ingredients, instructions, etc.) to the requested language. diff --git a/mealie/services/parser_services/openai/parser.py b/mealie/services/parser_services/openai/parser.py index 386a8ccfe..77e1c0229 100644 --- a/mealie/services/parser_services/openai/parser.py +++ b/mealie/services/parser_services/openai/parser.py @@ -82,7 +82,7 @@ class OpenAIParser(ABCIngredientParser): # re-combine chunks into one response responses_json = await asyncio.gather(*tasks) responses = [ - OpenAIIngredients.model_validate_json(response_json) for response_json in responses_json if responses_json + OpenAIIngredients.parse_openai_response(response_json) for response_json in responses_json if responses_json ] if not responses: raise Exception("No response from OpenAI") diff --git a/mealie/services/recipe/recipe_service.py b/mealie/services/recipe/recipe_service.py index 8df41e278..d2259728e 100644 --- a/mealie/services/recipe/recipe_service.py +++ b/mealie/services/recipe/recipe_service.py @@ -1,4 +1,5 @@ import json +import os import shutil from datetime import datetime, timezone from pathlib import Path @@ -11,24 +12,29 @@ from fastapi import UploadFile from slugify import slugify from mealie.core import exceptions +from mealie.core.config import get_app_settings +from mealie.core.dependencies.dependencies import get_temporary_path from mealie.lang.providers import Translator from mealie.pkgs import cache from mealie.repos.repository_factory import AllRepositories from mealie.repos.repository_generic import RepositoryGeneric +from mealie.schema.openai.recipe import OpenAIRecipe from mealie.schema.recipe.recipe import CreateRecipe, Recipe from mealie.schema.recipe.recipe_ingredient import RecipeIngredient +from mealie.schema.recipe.recipe_notes import RecipeNote from mealie.schema.recipe.recipe_settings import RecipeSettings from mealie.schema.recipe.recipe_step import RecipeStep from mealie.schema.recipe.recipe_timeline_events import RecipeTimelineEventCreate, TimelineEventType from mealie.schema.recipe.request_helpers import RecipeDuplicate from mealie.schema.user.user import GroupInDB, PrivateUser, UserRatingCreate from mealie.services._base_service import BaseService +from mealie.services.openai import OpenAIDataInjection, OpenAILocalImage, OpenAIService from mealie.services.recipe.recipe_data_service import RecipeDataService from .template_service import TemplateService -class RecipeService(BaseService): +class RecipeServiceBase(BaseService): def __init__(self, repos: AllRepositories, user: PrivateUser, group: GroupInDB, translator: Translator): self.repos = repos self.user = user @@ -39,6 +45,8 @@ class RecipeService(BaseService): super().__init__() + +class RecipeService(RecipeServiceBase): def _get_recipe(self, data: str | UUID, key: str | None = None) -> Recipe: recipe = self.repos.recipes.by_group(self.group.id).get_one(data, key) if recipe is None: @@ -250,6 +258,26 @@ class RecipeService(BaseService): return recipe + async def create_from_images(self, images: list[UploadFile], translate_language: str | None = None) -> Recipe: + openai_recipe_service = OpenAIRecipeService(self.repos, self.user, self.group, self.translator) + with get_temporary_path() as temp_path: + local_images: list[Path] = [] + for image in images: + with temp_path.joinpath(image.filename).open("wb") as buffer: + shutil.copyfileobj(image.file, buffer) + local_images.append(temp_path.joinpath(image.filename)) + + recipe_data = await openai_recipe_service.build_recipe_from_images( + local_images, translate_language=translate_language + ) + + recipe = self.create_one(recipe_data) + data_service = RecipeDataService(recipe.id) + + with open(local_images[0], "rb") as f: + data_service.write_image(f.read(), "webp") + return recipe + def duplicate_one(self, old_slug: str, dup_data: RecipeDuplicate) -> Recipe: """Duplicates a recipe and returns the new recipe.""" @@ -379,3 +407,67 @@ class RecipeService(BaseService): def render_template(self, recipe: Recipe, temp_dir: Path, template: str) -> Path: t_service = TemplateService(temp_dir) return t_service.render(recipe, template) + + +class OpenAIRecipeService(RecipeServiceBase): + def _convert_recipe(self, openai_recipe: OpenAIRecipe) -> Recipe: + return Recipe( + user_id=self.user.id, + group_id=self.user.group_id, + name=openai_recipe.name, + slug=slugify(openai_recipe.name), + description=openai_recipe.description, + recipe_yield=openai_recipe.recipe_yield, + total_time=openai_recipe.total_time, + prep_time=openai_recipe.prep_time, + perform_time=openai_recipe.perform_time, + recipe_ingredient=[ + RecipeIngredient(title=ingredient.title, note=ingredient.text) + for ingredient in openai_recipe.ingredients + if ingredient.text + ], + recipe_instructions=[ + RecipeStep(title=instruction.title, text=instruction.text) + for instruction in openai_recipe.instructions + if instruction.text + ], + notes=[RecipeNote(title=note.title or "", text=note.text) for note in openai_recipe.notes if note.text], + ) + + async def build_recipe_from_images(self, images: list[Path], translate_language: str | None) -> Recipe: + settings = get_app_settings() + if not (settings.OPENAI_ENABLED and settings.OPENAI_ENABLE_IMAGE_SERVICES): + raise ValueError("OpenAI image services are not available") + + openai_service = OpenAIService() + prompt = openai_service.get_prompt( + "recipes.parse-recipe-image", + data_injections=[ + OpenAIDataInjection( + description=( + "This is the JSON response schema. You must respond in valid JSON that follows this schema. " + "Your payload should be as compact as possible, eliminating unncessesary whitespace. " + "Any fields with default values which you do not populate should not be in the payload." + ), + value=OpenAIRecipe, + ) + ], + ) + + openai_images = [OpenAILocalImage(filename=os.path.basename(image), path=image) for image in images] + message = ( + f"Please extract the recipe from the {'images' if len(openai_images) > 1 else 'image'} provided." + "There should be exactly one recipe." + ) + + if translate_language: + message += f" Please translate the recipe to {translate_language}." + + response = await openai_service.get_response(prompt, message, images=openai_images, force_json_response=True) + try: + openai_recipe = OpenAIRecipe.parse_openai_response(response) + recipe = self._convert_recipe(openai_recipe) + except Exception as e: + raise ValueError("Unable to parse recipe from image") from e + + return recipe diff --git a/tests/integration_tests/user_recipe_tests/test_recipe_create_from_image.py b/tests/integration_tests/user_recipe_tests/test_recipe_create_from_image.py new file mode 100644 index 000000000..073970049 --- /dev/null +++ b/tests/integration_tests/user_recipe_tests/test_recipe_create_from_image.py @@ -0,0 +1,58 @@ +import json + +import pytest +from fastapi.testclient import TestClient + +from mealie.schema.openai.recipe import ( + OpenAIRecipe, + OpenAIRecipeIngredient, + OpenAIRecipeInstruction, + OpenAIRecipeNotes, +) +from mealie.services.openai import OpenAIService +from tests.utils import api_routes +from tests.utils.factories import random_int, random_string +from tests.utils.fixture_schemas import TestUser + + +def test_openai_create_recipe_from_image( + api_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + unique_user: TestUser, + test_image_jpg: str, +): + async def mock_get_response(self, prompt: str, message: str, *args, **kwargs) -> str | None: + data = OpenAIRecipe( + name=random_string(), + description=random_string(), + recipe_yield=random_string(), + total_time=random_string(), + prep_time=random_string(), + perform_time=random_string(), + ingredients=[OpenAIRecipeIngredient(text=random_string()) for _ in range(random_int(5, 10))], + instructions=[OpenAIRecipeInstruction(text=random_string()) for _ in range(1, random_int(5, 10))], + notes=[OpenAIRecipeNotes(text=random_string()) for _ in range(random_int(2, 5))], + ) + return data.model_dump_json() + + monkeypatch.setattr(OpenAIService, "get_response", mock_get_response) + with open(test_image_jpg, "rb") as f: + r = api_client.post( + api_routes.recipes_create_from_image, + files={"images": ("test_image_jpg.jpg", f, "image/jpeg")}, + data={"extension": "jpg"}, + headers=unique_user.token, + ) + assert r.status_code == 201 + + # since OpenAI is mocked, we don't need to validate the data, we just need to make sure a recipe is created, + # and that it has an image + slug: str = json.loads(r.text) + r = api_client.get(api_routes.recipes_slug(slug), headers=unique_user.token) + assert r.status_code == 200 + recipe_id = r.json()["id"] + + r = api_client.get( + api_routes.media_recipes_recipe_id_images_file_name(recipe_id, "original.webp"), headers=unique_user.token + ) + assert r.status_code == 200 diff --git a/tests/utils/api_routes/__init__.py b/tests/utils/api_routes/__init__.py index 351331ea0..39819833f 100644 --- a/tests/utils/api_routes/__init__.py +++ b/tests/utils/api_routes/__init__.py @@ -17,20 +17,14 @@ admin_email = "/api/admin/email" """`/api/admin/email`""" admin_groups = "/api/admin/groups" """`/api/admin/groups`""" -admin_logs = "/api/admin/logs" -"""`/api/admin/logs`""" admin_maintenance = "/api/admin/maintenance" """`/api/admin/maintenance`""" admin_maintenance_clean_images = "/api/admin/maintenance/clean/images" """`/api/admin/maintenance/clean/images`""" -admin_maintenance_clean_logs = "/api/admin/maintenance/clean/logs" -"""`/api/admin/maintenance/clean/logs`""" admin_maintenance_clean_recipe_folders = "/api/admin/maintenance/clean/recipe-folders" """`/api/admin/maintenance/clean/recipe-folders`""" admin_maintenance_clean_temp = "/api/admin/maintenance/clean/temp" """`/api/admin/maintenance/clean/temp`""" -admin_maintenance_logs = "/api/admin/maintenance/logs" -"""`/api/admin/maintenance/logs`""" admin_maintenance_storage = "/api/admin/maintenance/storage" """`/api/admin/maintenance/storage`""" admin_users = "/api/admin/users" @@ -47,6 +41,8 @@ app_about_startup_info = "/api/app/about/startup-info" """`/api/app/about/startup-info`""" app_about_theme = "/api/app/about/theme" """`/api/app/about/theme`""" +auth_logout = "/api/auth/logout" +"""`/api/auth/logout`""" auth_refresh = "/api/auth/refresh" """`/api/auth/refresh`""" auth_token = "/api/auth/token" @@ -143,6 +139,8 @@ recipes_bulk_actions_settings = "/api/recipes/bulk-actions/settings" """`/api/recipes/bulk-actions/settings`""" recipes_bulk_actions_tag = "/api/recipes/bulk-actions/tag" """`/api/recipes/bulk-actions/tag`""" +recipes_create_from_image = "/api/recipes/create-from-image" +"""`/api/recipes/create-from-image`""" recipes_create_from_zip = "/api/recipes/create-from-zip" """`/api/recipes/create-from-zip`""" recipes_create_url = "/api/recipes/create-url" @@ -212,11 +210,6 @@ def admin_groups_item_id(item_id): return f"{prefix}/admin/groups/{item_id}" -def admin_logs_num(num): - """`/api/admin/logs/{num}`""" - return f"{prefix}/admin/logs/{num}" - - def admin_users_item_id(item_id): """`/api/admin/users/{item_id}`""" return f"{prefix}/admin/users/{item_id}" @@ -362,6 +355,11 @@ def groups_webhooks_item_id(item_id): return f"{prefix}/groups/webhooks/{item_id}" +def groups_webhooks_item_id_test(item_id): + """`/api/groups/webhooks/{item_id}/test`""" + return f"{prefix}/groups/webhooks/{item_id}/test" + + def media_recipes_recipe_id_assets_file_name(recipe_id, file_name): """`/api/media/recipes/{recipe_id}/assets/{file_name}`""" return f"{prefix}/media/recipes/{recipe_id}/assets/{file_name}"