dev: Create a Python package, build Docker images from it (#4551)

Co-authored-by: Kuchenpirat <24235032+Kuchenpirat@users.noreply.github.com>
Co-authored-by: Michael Genson <71845777+michael-genson@users.noreply.github.com>
Co-authored-by: Hayden <64056131+hay-kot@users.noreply.github.com>

.dockerignore

@@ -6,7 +6,7 @@
 .idea
 .vscode
 
-__pycache__/
+**/__pycache__/
 *.py[cod]
 *$py.class
 *.so
@@ -25,9 +25,11 @@ venv
 
 */node_modules
 */dist
+/dist/
 */data/db
 */mealie/test
 */mealie/.temp
+/mealie/frontend/
 
 model.crfmodel
 

.github/workflows/e2e.yml

@@ -3,8 +3,15 @@ on:
   workflow_call:
 
 jobs:
+  build-package:
+    name: "Build Python package"
+    uses: ./.github/workflows/partial-package.yml
+    with:
+      tag: e2e
+
   test:
     timeout-minutes: 60
+    needs: build-package
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -18,11 +25,18 @@ jobs:
         cache-dependency-path: ./tests/e2e/yarn.lock
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
+    - name: Retrieve Python package
+      uses: actions/download-artifact@v4
+      with:
+        name: backend-dist
+        path: dist
     - name: Build Image
       uses: docker/build-push-action@v5
       with:
         file: ./docker/Dockerfile
         context: .
+        build-contexts: |
+          packages=dist
         push: false
         load: true
         tags: mealie:e2e

.github/workflows/nightly.yml

@@ -21,7 +21,7 @@ jobs:
     uses: ./.github/workflows/partial-backend.yml
 
   frontend-tests:
-    name: "Frontend and End-to-End Tests"
+    name: "Frontend Tests"
     uses: ./.github/workflows/partial-frontend.yml
 
   build-release:

.github/workflows/partial-backend.yml

@@ -1,4 +1,4 @@
-name: Backend Test/Lint
+name: Backend Lint and Test
 
 on:
   workflow_call:

.github/workflows/partial-builder.yml

@@ -16,7 +16,14 @@ on:
         required: true
 
 jobs:
+  build-package:
+    name: "Build Python package"
+    uses: ./.github/workflows/partial-package.yml
+    with:
+      tag: ${{ inputs.tag }}
+
   publish:
+    needs: build-package
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -35,18 +42,22 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Override __init__.py
-        run: |
-          echo "__version__ = \"${{ inputs.tag }}\"" > ./mealie/__init__.py
-
       - uses: depot/setup-action@v1
 
+      - name: Retrieve Python package
+        uses: actions/download-artifact@v4
+        with:
+          name: backend-dist
+          path: dist
+
       - name: Build and push Docker image, via Depot.dev
         uses: depot/build-push-action@v1
         with:
           project: srzjb6mhzm
           file: ./docker/Dockerfile
           context: .
+          build-contexts: |
+            packages=dist
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |

.github/workflows/partial-frontend.yml

@@ -1,4 +1,4 @@
-name: Frontend Build/Lin
+name: Frontend Lint and Test
 
 on:
   workflow_call:
@@ -41,37 +41,3 @@ jobs:
       - name: Run tests 🧪
         run: yarn test:ci
         working-directory: "frontend"
-
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout 🛎
-        uses: actions/checkout@v4
-
-      - name: Setup node env 🏗
-        uses: actions/setup-node@v4.0.0
-        with:
-          node-version: 16
-          check-latest: true
-
-      - name: Get yarn cache directory path 🛠
-        id: yarn-cache-dir-path
-        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
-
-      - name: Cache node_modules 📦
-        uses: actions/cache@v4
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - name: Install dependencies 👨🏻‍💻
-        run: yarn
-        working-directory: "frontend"
-
-      - name: Run Build 🚚
-        run: yarn build
-        working-directory: "frontend"

.github/workflows/partial-package.yml

@@ -0,0 +1,102 @@
+name: Package build
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        required: true
+        type: string
+
+jobs:
+  build-frontend:
+    name: Build frontend
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout 🛎
+        uses: actions/checkout@v4
+
+      - name: Setup node env 🏗
+        uses: actions/setup-node@v4.0.0
+        with:
+          node-version: 16
+          check-latest: true
+
+      - name: Get yarn cache directory path 🛠
+        id: yarn-cache-dir-path
+        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Cache node_modules 📦
+        uses: actions/cache@v4
+        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install dependencies 👨🏻‍💻
+        run: yarn
+        working-directory: "frontend"
+
+      - name: Run Build 🚚
+        run: yarn generate
+        working-directory: "frontend"
+
+      - name: Archive built frontend
+        uses: actions/upload-artifact@v4
+        with:
+          name: frontend-dist
+          path: frontend/dist
+          retention-days: 5
+
+  build-package:
+    name: Build Python package
+    needs: build-frontend
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Install Task
+        uses: arduino/setup-task@v2
+        with:
+          version: 3.x
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          plugins: |
+            poetry-plugin-export
+
+      - name: Retrieve built frontend
+        uses: actions/download-artifact@v4
+        with:
+          name: frontend-dist
+          path: mealie/frontend
+
+      - name: Override __init__.py
+        run: |
+          echo "__version__ = \"${{ inputs.tag }}\"" > ./mealie/__init__.py
+
+      - name: Build package and requirements.txt
+        env:
+          SKIP_PACKAGE_DEPS: true
+        run: |
+          task py:package
+
+      - name: Archive built package
+        uses: actions/upload-artifact@v4
+        with:
+          name: backend-dist
+          path: dist
+          retention-days: 5

.github/workflows/pull-requests.yml

@@ -19,7 +19,7 @@ jobs:
     uses: ./.github/workflows/partial-backend.yml
 
   frontend-tests:
-    name: "Frontend and End-to-End Tests"
+    name: "Frontend Tests"
     uses: ./.github/workflows/partial-frontend.yml
 
   container-scanning:

.github/workflows/release.yml

@@ -10,7 +10,7 @@ jobs:
     uses: ./.github/workflows/partial-backend.yml
 
   frontend-tests:
-    name: "Frontend and End-to-End Tests"
+    name: "Frontend Tests"
     uses: ./.github/workflows/partial-frontend.yml
 
   build-release:

.gitignore

@@ -52,7 +52,7 @@ pnpm-debug.log*
 env/
 build/
 develop-eggs/
-
+/dist/
 downloads/
 eggs/
 .eggs/
@@ -66,6 +66,9 @@ wheels/
 .installed.cfg
 *.egg
 
+# frontend copied into Python module for packaging purposes
+/mealie/frontend/
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

Taskfile.yml

@@ -41,14 +41,25 @@ tasks:
   setup:ui:
     desc: setup frontend dependencies
     dir: frontend
+    run: once
     cmds:
       - yarn install
+    sources:
+      - package.json
+      - yarn.lock
+    generates:
+      - node_modules/**
 
   setup:py:
     desc: setup python dependencies
+    run: once
     cmds:
       - poetry install --with main,dev,postgres
       - poetry run pre-commit install
+    sources:
+      - poetry.lock
+      - pyproject.toml
+      - .pre-commit-config.yaml
 
   setup:model:
     desc: setup nlp model
@@ -131,6 +142,63 @@ tasks:
       - poetry run coverage html
       - open htmlcov/index.html
 
+  py:package:copy-frontend:
+    desc: copy the frontend files into the Python package
+    internal: true
+    deps:
+      - ui:generate
+    cmds:
+      - rm -rf mealie/frontend
+      - cp -a frontend/dist mealie/frontend
+    sources:
+      - frontend/dist/**
+    generates:
+      - mealie/frontend/**
+
+  py:package:generate-requirements:
+    desc: Generate requirements file to pin all packages, effectively a "pip freeze" before installation begins
+    internal: true
+    cmds:
+      - poetry export -n --only=main --extras=pgsql --output=dist/requirements.txt
+      # Include mealie in the requirements, hashing the package that was just built to ensure it's the one installed
+      - echo "mealie[pgsql]=={{.MEALIE_VERSION}} \\" >> dist/requirements.txt
+      - poetry run pip hash dist/mealie-{{.MEALIE_VERSION}}-py3-none-any.whl | tail -n1 | tr -d '\n' >> dist/requirements.txt
+      - echo " \\" >> dist/requirements.txt
+      - poetry run pip hash dist/mealie-{{.MEALIE_VERSION}}.tar.gz | tail -n1 >> dist/requirements.txt
+    vars:
+      MEALIE_VERSION:
+         sh: poetry version --short
+    sources:
+      - poetry.lock
+      - pyproject.toml
+      - dist/mealie-*.whl
+      - dist/mealie-*.tar.gz
+    generates:
+      - dist/requirements.txt
+
+  py:package:deps-parallel:
+    desc: Run py:package dependencies in parallel
+    internal: true
+    deps:
+      - setup:py
+      - py:package:copy-frontend
+
+  py:package:deps:
+    desc: Dependencies of py:package, skippable by setting SKIP_PACKAGE_DEPS=true
+    internal: true
+    cmds:
+      - task: py:package:deps-parallel
+    status:
+      - '{{ .SKIP_PACKAGE_DEPS | default "false"}}'
+
+  py:package:
+    desc: builds Python packages (sdist and wheel) in top-level dist directory
+    deps:
+      - py:package:deps
+    cmds:
+      - poetry build -n --output=dist
+      - task: py:package:generate-requirements
+
   py:
     desc: runs the backend server
     cmds:
@@ -160,6 +228,14 @@ tasks:
     cmds:
       - yarn build
 
+  ui:generate:
+    desc: generates a static version of the frontend in frontend/dist
+    dir: frontend
+    deps:
+      - setup:ui
+    cmds:
+      - yarn generate
+
   ui:lint:
     desc: runs the frontend linter
     dir: frontend
@@ -184,6 +260,16 @@ tasks:
     cmds:
       - yarn run dev
 
+  docker:build-from-package:
+    desc: Builds the Docker image from the existing Python package in dist/
+    deps:
+       - py:package
+    cmds:
+       - docker build --tag mealie:dev --file docker/Dockerfile --build-arg COMMIT={{.GIT_COMMIT}} --build-context packages=dist .
+    vars:
+      GIT_COMMIT:
+        sh: git rev-parse HEAD
+
   docker:prod:
     desc: builds and runs the production docker image locally
     dir: docker

docker/Dockerfile

@@ -1,8 +1,11 @@
-FROM node:16 as builder
+###############################################
+# Frontend Build
+###############################################
+FROM node:16 AS frontend-builder
 
-WORKDIR /app
+WORKDIR /frontend
 
-COPY ./frontend .
+COPY frontend .
 
 RUN yarn install \
     --prefer-offline \
@@ -26,14 +29,10 @@ ENV PYTHONUNBUFFERED=1 \
     PIP_NO_CACHE_DIR=off \
     PIP_DISABLE_PIP_VERSION_CHECK=on \
     PIP_DEFAULT_TIMEOUT=100 \
-    POETRY_HOME="/opt/poetry" \
-    POETRY_VIRTUALENVS_IN_PROJECT=true \
-    POETRY_NO_INTERACTION=1 \
-    PYSETUP_PATH="/opt/pysetup" \
-    VENV_PATH="/opt/pysetup/.venv"
+    VENV_PATH="/opt/mealie"
 
-# prepend poetry and venv to path
-ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+# prepend venv to path
+ENV PATH="$VENV_PATH/bin:$PATH"
 
 # create user account
 RUN useradd -u 911 -U -d $MEALIE_HOME -s /bin/bash abc \
@@ -41,31 +40,81 @@ RUN useradd -u 911 -U -d $MEALIE_HOME -s /bin/bash abc \
     && mkdir $MEALIE_HOME
 
 ###############################################
-# Builder Image
+# Backend Package Build
 ###############################################
-FROM python-base as builder-base
+FROM python-base AS backend-builder
 RUN apt-get update \
     && apt-get install --no-install-recommends -y \
     curl \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV POETRY_HOME="/opt/poetry" \
+    POETRY_NO_INTERACTION=1
+
+# prepend poetry to path
+ENV PATH="$POETRY_HOME/bin:$PATH"
+
+# install poetry - respects $POETRY_VERSION & $POETRY_HOME
+ENV POETRY_VERSION=2.0.1
+RUN curl -sSL https://install.python-poetry.org | python3 -
+
+# install poetry plugins needed to build the package
+RUN poetry self add "poetry-plugin-export>=1.9"
+
+WORKDIR /mealie
+
+# copy project files here to ensure they will be cached.
+COPY poetry.lock pyproject.toml ./
+COPY mealie ./mealie
+
+# Copy frontend to package it into the wheel
+COPY --from=frontend-builder /frontend/dist ./mealie/frontend
+
+# Build the source and binary package
+RUN poetry build --output=dist
+
+# Create the requirements file, which is used to install the built package and
+# its pinned dependencies later. mealie is included to ensure the built one is
+# what's installed.
+RUN export MEALIE_VERSION=$(poetry version --short) \
+    && poetry export --only=main --extras=pgsql --output=dist/requirements.txt \
+    && echo "mealie[pgsql]==$MEALIE_VERSION \\" >> dist/requirements.txt \
+    && poetry run pip hash dist/mealie-$MEALIE_VERSION-py3-none-any.whl | tail -n1 | tr -d '\n' >> dist/requirements.txt \
+    && echo " \\" >> dist/requirements.txt \
+    && poetry run pip hash dist/mealie-$MEALIE_VERSION.tar.gz | tail -n1 >> dist/requirements.txt
+
+###############################################
+# Package Container
+# Only role is to hold the packages, or be overriden by a --build-context flag.
+###############################################
+FROM scratch AS packages
+COPY --from=backend-builder /mealie/dist /
+
+###############################################
+# Python Virtual Environment Build
+###############################################
+# Install packages required to build the venv, in parallel to building the wheel
+FROM python-base AS venv-builder-base
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
     build-essential \
     libpq-dev \
     libwebp-dev \
     # LDAP Dependencies
     libsasl2-dev libldap2-dev libssl-dev \
     gnupg gnupg2 gnupg1 \
-    && rm -rf /var/lib/apt/lists/* \
-    && pip install -U --no-cache-dir pip
+    && rm -rf /var/lib/apt/lists/*
+RUN python3 -m venv --upgrade-deps $VENV_PATH
 
-# install poetry - respects $POETRY_VERSION & $POETRY_HOME
-ENV POETRY_VERSION=1.3.1
-RUN curl -sSL https://install.python-poetry.org | python3 -
+# Install the wheel and all dependencies into the venv
+FROM venv-builder-base AS venv-builder
 
-# copy project requirement files here to ensure they will be cached.
-WORKDIR $PYSETUP_PATH
-COPY ./poetry.lock ./pyproject.toml ./
+# Copy built package (wheel) and its dependency requirements
+COPY --from=packages * /dist/
 
-# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
-RUN poetry install -E pgsql --only main
+# Install the wheel with exact versions of dependencies into the venv
+RUN . $VENV_PATH/bin/activate \
+    && pip install --require-hashes -r /dist/requirements.txt --find-links /dist
 
 ###############################################
 # CRFPP Image
@@ -96,39 +145,25 @@ RUN apt-get update \
 # create directory used for Docker Secrets
 RUN mkdir -p /run/secrets
 
-# copying poetry and venv into image
-COPY --from=builder-base $POETRY_HOME $POETRY_HOME
-COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
-
+# copy CRF++ and add it to the library path
 ENV LD_LIBRARY_PATH=/usr/local/lib
 COPY --from=crfpp /usr/local/lib/ /usr/local/lib
 COPY --from=crfpp /usr/local/bin/crf_learn /usr/local/bin/crf_learn
 COPY --from=crfpp /usr/local/bin/crf_test /usr/local/bin/crf_test
 
-# copy backend
-COPY ./mealie $MEALIE_HOME/mealie
-COPY ./poetry.lock ./pyproject.toml $MEALIE_HOME/
+# Copy venv into image. It contains a fully-installed mealie backend and frontend.
+COPY --from=venv-builder $VENV_PATH $VENV_PATH
 
-# venv already has runtime deps installed we get a quicker install
-WORKDIR $MEALIE_HOME
-RUN . $VENV_PATH/bin/activate && poetry install -E pgsql --only main
-WORKDIR /
 
 # Grab CRF++ Model Release
-RUN python $MEALIE_HOME/mealie/scripts/install_model.py
+RUN python -m mealie.scripts.install_model
 
 VOLUME [ "$MEALIE_HOME/data/" ]
 ENV APP_PORT=9000
 
 EXPOSE ${APP_PORT}
 
-HEALTHCHECK CMD python $MEALIE_HOME/mealie/scripts/healthcheck.py || exit 1
-
-# ----------------------------------
-# Copy Frontend
-
-ENV STATIC_FILES=/spa/static
-COPY --from=builder /app/dist  ${STATIC_FILES}
+HEALTHCHECK CMD python -m mealie.scripts.healthcheck || exit 1
 
 ENV HOST 0.0.0.0
 

docker/entry.sh

@@ -32,7 +32,7 @@ init() {
     cd /app
 
     # Activate our virtual environment here
-    . /opt/pysetup/.venv/bin/activate
+    . /opt/mealie/bin/activate
 }
 
 change_user
@@ -41,4 +41,4 @@ init
 # Start API
 HOST_IP=`/sbin/ip route|awk '/default/ { print $3 }'`
 
-exec python /app/mealie/main.py
+exec mealie

docs/docs/contributors/developers-guide/building-packages.md

@@ -0,0 +1,40 @@
+# Building Packages
+
+Released packages are [built and published via GitHub actions](maintainers.md#drafting-releases).
+
+## Python packages
+
+To build Python packages locally for testing, use [`task`](starting-dev-server.md#without-dev-containers). After installing `task`, run `task py:package` to perform all the steps needed to build the package and a requirements file. To do it manually, run:
+```sh
+pushd frontend
+yarnpkg install
+yarnpkg generate
+popd
+rm -r mealie/frontend
+cp -a frontend/dist mealie/frontend
+poetry build
+poetry export -n --only=main --extras=pgsql --output=dist/requirements.txt
+MEALIE_VERSION=$(poetry version --short)
+echo "mealie[pgsql]==${MEALIE_VERSION} \\" >> dist/requirements.txt
+poetry run pip hash dist/mealie-${MEALIE_VERSION}-py3-none-any.whl | tail -n1 | tr -d '\n' >> dist/requirements.txt
+echo " \\" >> dist/requirements.txt
+poetry run pip hash dist/mealie-${MEALIE_VERSION}.tar.gz | tail -n1 >> dist/requirements.txt
+```
+
+The Python package can be installed with all of its dependencies pinned to the versions tested by the developers with:
+```sh
+pip3 install -r dist/requirements.txt --find-links dist
+```
+
+To install with the latest but still compatible dependency versions, instead run `pip3 install dist/mealie-$VERSION-py3-none-any.whl` (where `$VERSION` is the version of mealie to install).
+
+## Docker image
+One way to build the Docker image is to run the following command in the project root directory:
+```sh
+docker build --tag mealie:dev --file docker/Dockerfile --build-arg COMMIT=$(git rev-parse HEAD) .
+```
+
+The Docker image can be built from the pre-built Python packages with the task command `task docker:build-from-package`. This is equivalent to:
+```sh
+docker build --tag mealie:dev --file docker/Dockerfile --build-arg COMMIT=$(git rev-parse HEAD) --build-context packages=dist .
+```

docs/mkdocs.yml

@@ -99,6 +99,7 @@ nav:
       - Non-Code: "contributors/non-coders.md"
       - Translating: "contributors/translating.md"
       - Developers Guide:
+          - Building Packages: "contributors/developers-guide/building-packages.md"
           - Code Contributions: "contributors/developers-guide/code-contributions.md"
           - Dev Getting Started: "contributors/developers-guide/starting-dev-server.md"
           - Database Changes: "contributors/developers-guide/database-changes.md"

mealie/core/settings/settings.py

@@ -12,6 +12,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 from mealie.core.settings.themes import Theme
 
 from .db_providers import AbstractDBProvider, db_provider_factory
+from .static import PACKAGE_DIR
 
 
 class ScheduleTime(NamedTuple):
@@ -109,7 +110,7 @@ class AppSettings(AppLoggingSettings):
     BASE_URL: str = "http://localhost:8080"
     """trailing slashes are trimmed (ex. `http://localhost:8080/` becomes ``http://localhost:8080`)"""
 
-    STATIC_FILES: str = ""
+    STATIC_FILES: str = str(PACKAGE_DIR / "frontend")
     """path to static files directory (ex. `mealie/dist`)"""
 
     IS_DEMO: bool = False

mealie/core/settings/static.py

@@ -5,4 +5,5 @@ from mealie import __version__
 APP_VERSION = __version__
 
 CWD = Path(__file__).parent
+PACKAGE_DIR = CWD.parent.parent
 BASE_DIR = CWD.parent.parent.parent

mealie/main.py

@@ -6,7 +6,7 @@ from mealie.core.logger.config import log_config
 
 def main():
     uvicorn.run(
-        "app:app",
+        "mealie.app:app",
         host=settings.API_HOST,
         port=settings.API_PORT,
         log_level=settings.LOG_LEVEL.lower(),

pyproject.toml

@@ -4,9 +4,13 @@ description = "A Recipe Manager"
 license = "AGPL"
 name = "mealie"
 version = "2.6.0"
+include = [
+    # Explicit include to override .gitignore when packaging the frontend
+    { path = "mealie/frontend/**/*", format = ["sdist", "wheel"] }
+]
 
 [tool.poetry.scripts]
-start = "mealie.app:main"
+mealie = "mealie.main:main"
 
 [tool.poetry.dependencies]
 Jinja2 = "^3.1.2"