From cb9008bb5c165409861b3e5762b62393f99f569b Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Tue, 11 Feb 2025 09:54:32 -0600 Subject: [PATCH] fix: shorten indexes to fix issues with index limits (#5045) --- .vscode/settings.json | 4 + ..._7cf3054cbbcc_remove_instructions_index.py | 146 ++++++++++++++++++ mealie/db/models/_model_base.py | 4 +- mealie/db/models/recipe/instruction.py | 4 +- 4 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 mealie/alembic/versions/2025-02-09-15.31.00_7cf3054cbbcc_remove_instructions_index.py diff --git a/.vscode/settings.json b/.vscode/settings.json index e4bfdeb9d..b842cd45e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -60,5 +60,9 @@ }, "[vue]": { "editor.formatOnSave": false + }, + "[python]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "charliermarsh.ruff" } } diff --git a/mealie/alembic/versions/2025-02-09-15.31.00_7cf3054cbbcc_remove_instructions_index.py b/mealie/alembic/versions/2025-02-09-15.31.00_7cf3054cbbcc_remove_instructions_index.py new file mode 100644 index 000000000..f6fbcca9d --- /dev/null +++ b/mealie/alembic/versions/2025-02-09-15.31.00_7cf3054cbbcc_remove_instructions_index.py @@ -0,0 +1,146 @@ +"""remove instructions index + +Revision ID: 7cf3054cbbcc +Revises: b9e516e2d3b3 +Create Date: 2025-02-09 15:31:00.772295 + +""" + +import sqlalchemy as sa +from sqlalchemy import orm +from alembic import op +from mealie.db.models._model_utils.guid import GUID +from mealie.core.root_logger import get_logger + +# revision identifiers, used by Alembic. +revision = "7cf3054cbbcc" +down_revision: str | None = "b9e516e2d3b3" +branch_labels: str | tuple[str, ...] | None = None +depends_on: str | tuple[str, ...] | None = None + +logger = get_logger() + + +class SqlAlchemyBase(orm.DeclarativeBase): + @classmethod + def normalized_fields(cls) -> list[orm.InstrumentedAttribute]: + return [] + + +class RecipeModel(SqlAlchemyBase): + __tablename__ = "recipes" + + id: orm.Mapped[GUID] = orm.mapped_column(GUID, primary_key=True, default=GUID.generate) + name_normalized: orm.Mapped[str] = orm.mapped_column(sa.String, nullable=False, index=True) + description_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + + @classmethod + def normalized_fields(cls): + return [cls.name_normalized, cls.description_normalized] + + +class RecipeIngredientModel(SqlAlchemyBase): + __tablename__ = "recipes_ingredients" + + id: orm.Mapped[int] = orm.mapped_column(sa.Integer, primary_key=True) + note_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + original_text_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + + @classmethod + def normalized_fields(cls): + return [cls.note_normalized, cls.original_text_normalized] + + +class IngredientFoodModel(SqlAlchemyBase): + __tablename__ = "ingredient_foods" + id: orm.Mapped[GUID] = orm.mapped_column(GUID, primary_key=True, default=GUID.generate) + name_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + plural_name_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + + @classmethod + def normalized_fields(cls): + return [cls.name_normalized, cls.plural_name_normalized] + + +class IngredientFoodAliasModel(SqlAlchemyBase): + __tablename__ = "ingredient_foods_aliases" + id: orm.Mapped[GUID] = orm.mapped_column(GUID, primary_key=True, default=GUID.generate) + name_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + + @classmethod + def normalized_fields(cls): + return [cls.name_normalized] + + +class IngredientUnitModel(SqlAlchemyBase): + __tablename__ = "ingredient_units" + id: orm.Mapped[GUID] = orm.mapped_column(GUID, primary_key=True, default=GUID.generate) + name_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + plural_name_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + abbreviation_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + plural_abbreviation_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + + @classmethod + def normalized_fields(cls): + return [ + cls.name_normalized, + cls.plural_name_normalized, + cls.abbreviation_normalized, + cls.plural_abbreviation_normalized, + ] + + +class IngredientUnitAliasModel(SqlAlchemyBase): + __tablename__ = "ingredient_units_aliases" + id: orm.Mapped[GUID] = orm.mapped_column(GUID, primary_key=True, default=GUID.generate) + name_normalized: orm.Mapped[str | None] = orm.mapped_column(sa.String, index=True) + + @classmethod + def normalized_fields(cls): + return [cls.name_normalized] + + +def truncate_normalized_fields() -> None: + bind = op.get_bind() + session = orm.Session(bind=bind) + + models: list[type[SqlAlchemyBase]] = [ + RecipeModel, + RecipeIngredientModel, + IngredientFoodModel, + IngredientFoodAliasModel, + IngredientUnitModel, + IngredientUnitAliasModel, + ] + + for model in models: + for record in session.query(model).all(): + for field in model.normalized_fields(): + if not (field_value := getattr(record, field.key)): + continue + + setattr(record, field.key, field_value[:255]) + + try: + session.commit() + except Exception: + logger.exception(f"Failed to truncate normalized fields for {model.__name__}") + session.rollback() + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("recipe_instructions", schema=None) as batch_op: + batch_op.drop_index("ix_recipe_instructions_text") + + # ### end Alembic commands ### + + truncate_normalized_fields() + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("recipe_instructions", schema=None) as batch_op: + batch_op.create_index("ix_recipe_instructions_text", ["text"], unique=False) + + # ### end Alembic commands ### diff --git a/mealie/db/models/_model_base.py b/mealie/db/models/_model_base.py index fd5e68c80..4298a838b 100644 --- a/mealie/db/models/_model_base.py +++ b/mealie/db/models/_model_base.py @@ -18,7 +18,9 @@ class SqlAlchemyBase(DeclarativeBase): @classmethod def normalize(cls, val: str) -> str: - return unidecode(val).lower().strip() + # We cap the length to 255 to prevent indexes from being too long; see: + # https://www.postgresql.org/docs/current/btree.html + return unidecode(val).lower().strip()[:255] class BaseMixins: diff --git a/mealie/db/models/recipe/instruction.py b/mealie/db/models/recipe/instruction.py index 70f7253ec..395b5dba9 100644 --- a/mealie/db/models/recipe/instruction.py +++ b/mealie/db/models/recipe/instruction.py @@ -23,8 +23,8 @@ class RecipeInstruction(SqlAlchemyBase): recipe_id: Mapped[GUID | None] = mapped_column(GUID, ForeignKey("recipes.id"), index=True) position: Mapped[int | None] = mapped_column(Integer, index=True) type: Mapped[str | None] = mapped_column(String, default="") - title: Mapped[str | None] = mapped_column(String) # This is the section title!!! - text: Mapped[str | None] = mapped_column(String, index=True) + title: Mapped[str | None] = mapped_column(String) # This is the section title + text: Mapped[str | None] = mapped_column(String) summary: Mapped[str | None] = mapped_column(String) ingredient_references: Mapped[list[RecipeIngredientRefLink]] = orm.relationship(