fix: More Backup Restore Fixes (#2859)

* refactor normalized search migration to use dummy default

* changed group slug migration to use raw SQL

* updated comment

* added tests with anonymized backups (currently failing)

* typo

* fixed LDAP enum in test data

* fix for adding label settings across groups

* add migration data fixes

* fix shopping list label settings test

* re-run db init instead of just running alembic migration, to include fixes

* intentionally broke SQLAlchemy GUID handling

* safely convert between GUID types in different databases

* restore original test data after testing backup restores

* added missing group name update to migration

alembic/versions/2023-02-14-20.45.41_5ab195a474eb_add_normalized_search_properties.py

@@ -7,12 +7,11 @@ Create Date: 2023-02-14 20:45:41.102571
 """
 import sqlalchemy as sa
 from sqlalchemy import orm, select
-from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 from text_unidecode import unidecode
 
 import mealie.db.migration_types
 from alembic import op
-
 from mealie.db.models._model_utils import GUID
 
 # revision identifiers, used by Alembic.
@@ -52,30 +51,46 @@ def do_data_migration():
     session = orm.Session(bind=bind)
 
     recipes = session.execute(select(RecipeModel)).scalars().all()
-    ingredients = session.execute(select(RecipeIngredient)).scalars().all()
     for recipe in recipes:
         if recipe.name is not None:
-            recipe.name_normalized = unidecode(recipe.name).lower().strip()
+            session.execute(
+                sa.text(
+                    f"UPDATE {RecipeModel.__tablename__} SET name_normalized=:name_normalized WHERE id=:id"
+                ).bindparams(name_normalized=unidecode(recipe.name).lower().strip(), id=recipe.id)
+            )
 
         if recipe.description is not None:
-            recipe.description_normalized = unidecode(recipe.description).lower().strip()
-        session.add(recipe)
+            session.execute(
+                sa.text(
+                    f"UPDATE {RecipeModel.__tablename__} SET description_normalized=:description_normalized WHERE id=:id"
+                ).bindparams(description_normalized=unidecode(recipe.description).lower().strip(), id=recipe.id)
+            )
 
+    ingredients = session.execute(select(RecipeIngredient)).scalars().all()
     for ingredient in ingredients:
         if ingredient.note is not None:
-            ingredient.note_normalized = unidecode(ingredient.note).lower().strip()
+            session.execute(
+                sa.text(
+                    f"UPDATE {RecipeIngredient.__tablename__} SET note_normalized=:note_normalized WHERE id=:id"
+                ).bindparams(note_normalized=unidecode(ingredient.note).lower().strip(), id=ingredient.id)
+            )
 
         if ingredient.original_text is not None:
-            ingredient.original_text_normalized = unidecode(ingredient.original_text).lower().strip()
-        session.add(ingredient)
+            session.execute(
+                sa.text(
+                    f"UPDATE {RecipeIngredient.__tablename__} SET original_text_normalized=:original_text_normalized WHERE id=:id"
+                ).bindparams(
+                    original_text_normalized=unidecode(ingredient.original_text).lower().strip(), id=ingredient.id
+                )
+            )
     session.commit()
 
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
 
-    # Set column to nullable first, since we do not have values here yet
-    op.add_column("recipes", sa.Column("name_normalized", sa.String(), nullable=True))
+    # Set column default first, since we do not have values here yet
+    op.add_column("recipes", sa.Column("name_normalized", sa.String(), nullable=False, server_default=""))
     op.add_column("recipes", sa.Column("description_normalized", sa.String(), nullable=True))
     op.drop_index("ix_recipes_description", table_name="recipes")
     op.drop_index("ix_recipes_name", table_name="recipes")
@@ -95,9 +110,9 @@ def upgrade():
         unique=False,
     )
     do_data_migration()
-    # Make recipes.name_normalized not nullable now that column should be filled for all rows
+    # Remove server default now that column should be filled for all rows
     with op.batch_alter_table("recipes", schema=None) as batch_op:
-        batch_op.alter_column("name_normalized", nullable=False, existing_type=sa.String())
+        batch_op.alter_column("name_normalized", existing_type=sa.String(), server_default=None)
     # ### end Alembic commands ###
 
 

alembic/versions/2023-02-21-22.03.19_b04a08da2108_added_shopping_list_label_settings.py

@@ -24,10 +24,10 @@ depends_on = None
 
 def populate_shopping_lists_multi_purpose_labels(shopping_lists_multi_purpose_labels_table: sa.Table, session: Session):
     shopping_lists = session.query(ShoppingList).all()
-    labels = session.query(MultiPurposeLabel).all()
 
     shopping_lists_labels_data: list[dict] = []
     for shopping_list in shopping_lists:
+        labels = session.query(MultiPurposeLabel).filter(MultiPurposeLabel.group_id == ShoppingList.group_id).all()
         for i, label in enumerate(labels):
             shopping_lists_labels_data.append(
                 {"id": uuid4(), "shopping_list_id": shopping_list.id, "label_id": label.id, "position": i}

alembic/versions/2023-08-06-21.00.34_04ac51cbe9a4_added_group_slug.py

@@ -24,17 +24,22 @@ def populate_group_slugs(session: Session):
     seen_slugs: set[str] = set()
     for group in groups:
         original_name = group.name
+        new_name = original_name
         attempts = 0
         while True:
-            slug = slugify(group.name)
+            slug = slugify(new_name)
             if slug not in seen_slugs:
                 break
 
             attempts += 1
-            group.name = f"{original_name} ({attempts})"
+            new_name = f"{original_name} ({attempts})"
 
         seen_slugs.add(slug)
-        group.slug = slug
+        session.execute(
+            sa.text(f"UPDATE {Group.__tablename__} SET name=:name, slug=:slug WHERE id=:id").bindparams(
+                name=new_name, slug=slug, id=group.id
+            )
+        )
 
     session.commit()