fix: More Backup Restore Fixes (#2859)

* refactor normalized search migration to use dummy default

* changed group slug migration to use raw SQL

* updated comment

* added tests with anonymized backups (currently failing)

* typo

* fixed LDAP enum in test data

* fix for adding label settings across groups

* add migration data fixes

* fix shopping list label settings test

* re-run db init instead of just running alembic migration, to include fixes

* intentionally broke SQLAlchemy GUID handling

* safely convert between GUID types in different databases

* restore original test data after testing backup restores

* added missing group name update to migration

tests/data/__init__.py

@@ -4,6 +4,15 @@ CWD = Path(__file__).parent
 
 locale_dir = CWD / "locale"
 
+backup_version_44e8d670719d = CWD / "backups/backup_version_44e8d670719d.zip"
+"""44e8d670719d: add extras to shopping lists, list items, and ingredient foods"""
+
+backup_version_ba1e4a6cfe99 = CWD / "backups/backup_version_ba1e4a6cfe99.zip"
+"""ba1e4a6cfe99: added plural names and alias tables for foods and units"""
+
+backup_version_bcfdad6b7355 = CWD / "backups/backup_version_bcfdad6b7355.zip"
+"""bcfdad6b7355: remove tool name and slug unique contraints"""
+
 migrations_paprika = CWD / "migrations/paprika.zip"
 
 migrations_chowdown = CWD / "migrations/chowdown.zip"

tests/unit_tests/services_tests/backup_v2_tests/test_backup_v2.py

@@ -1,8 +1,18 @@
 import filecmp
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
+import pytest
+from sqlalchemy.orm import Session
+
+import tests.data as test_data
 from mealie.core.config import get_app_settings
+from mealie.db.db_setup import session_context
+from mealie.db.models.group import Group
+from mealie.db.models.group.shopping_list import ShoppingList
+from mealie.db.models.labels import MultiPurposeLabel
+from mealie.db.models.recipe.ingredient import IngredientFoodModel, IngredientUnitModel
+from mealie.db.models.recipe.recipe import RecipeModel
 from mealie.services.backups_v2.alchemy_exporter import AlchemyExporter
 from mealie.services.backups_v2.backup_file import BackupFile
 from mealie.services.backups_v2.backup_v2 import BackupV2
@@ -56,3 +66,88 @@ def test_database_restore():
 
     for s1, s2 in zip(snapshop_1, snapshop_2):
         assert snapshop_1[s1].sort(key=dict_sorter) == snapshop_2[s2].sort(key=dict_sorter)
+
+
+@pytest.mark.parametrize(
+    "backup_path",
+    [
+        test_data.backup_version_44e8d670719d,
+        test_data.backup_version_ba1e4a6cfe99,
+        test_data.backup_version_bcfdad6b7355,
+    ],
+    ids=[
+        "44e8d670719d: add extras to shopping lists, list items, and ingredient foods",
+        "ba1e4a6cfe99: added plural names and alias tables for foods and units",
+        "bcfdad6b7355: remove tool name and slug unique contraints",
+    ],
+)
+def test_database_restore_data(backup_path: Path):
+    """
+    This tests real user backups to make sure the data is restored correctly. The data has been anonymized, but
+    relationships and data types should be preserved.
+
+    This test should verify all migrations that do some sort of database manipulation (e.g. populating a new column).
+    If a new migration is added that does any sort of data manipulation, this test should be updated.
+    """
+
+    settings = get_app_settings()
+    backup_v2 = BackupV2(settings.DB_URL)
+
+    # create a backup of the existing data so we can restore it later
+    original_data_backup = backup_v2.backup()
+
+    try:
+        assert backup_path.exists()
+        backup_v2.restore(backup_path)
+
+        # make sure migrations populated data successfully
+        with session_context() as session:
+            session = cast(Session, session)
+
+            groups = session.query(Group).all()
+            recipes = session.query(RecipeModel).all()
+            shopping_lists = session.query(ShoppingList).all()
+            labels = session.query(MultiPurposeLabel).all()
+
+            foods = session.query(IngredientFoodModel).all()
+            units = session.query(IngredientUnitModel).all()
+
+            # 2023-02-14-20.45.41_5ab195a474eb_add_normalized_search_properties
+            for recipe in recipes:
+                if recipe.name:
+                    assert recipe.name_normalized
+                if recipe.description:
+                    assert recipe.description_normalized
+
+                for ingredient in recipe.recipe_ingredient:
+                    if ingredient.note:
+                        assert ingredient.note_normalized
+                    if ingredient.original_text:
+                        assert ingredient.original_text_normalized
+
+            # 2023-02-21-22.03.19_b04a08da2108_added_shopping_list_label_settings
+            for shopping_list in shopping_lists:
+                group_labels = [label for label in labels if label.group_id == shopping_list.group_id]
+                assert len(shopping_list.label_settings) == len(group_labels)
+                for label_setting, label in zip(
+                    sorted(shopping_list.label_settings, key=lambda x: x.label.id),
+                    sorted(group_labels, key=lambda x: x.id),
+                    strict=True,
+                ):
+                    assert label_setting.label == label
+
+            # 2023-08-06-21.00.34_04ac51cbe9a4_added_group_slug
+            for group in groups:
+                assert group.slug
+
+            # 2023-09-01-14.55.42_0341b154f79a_added_normalized_unit_and_food_names
+            for food in foods:
+                if food.name:
+                    assert food.name_normalized
+
+            for unit in units:
+                assert unit.name_normalized
+                if unit.abbreviation:
+                    assert unit.abbreviation_normalized
+    finally:
+        backup_v2.restore(original_data_backup)