feature: query filter support for common SQL keywords (#2366)

* added support for SQL keywords IS, IN, LIKE, NOT
deprecated datetime workaround for "<> null"
updated frontend reference for "<> null" to "IS NOT NULL"

* tests

* refactored query filtering to leverage orm

* added CONTAINS ALL keyword

* tests

* fixed bug where "and" or "or" was in an attr name

* more tests

* linter fixes

* TIL this works

tests/unit_tests/repository_tests/test_pagination.py

@@ -1,5 +1,6 @@
 import time
 from collections import defaultdict
+from datetime import datetime
 from random import randint
 from urllib.parse import parse_qsl, urlsplit
 
@@ -9,7 +10,10 @@ from humps import camelize
 
 from mealie.repos.repository_factory import AllRepositories
 from mealie.repos.repository_units import RepositoryUnit
+from mealie.schema.recipe import Recipe
+from mealie.schema.recipe.recipe_category import CategorySave, TagSave
 from mealie.schema.recipe.recipe_ingredient import IngredientUnit, SaveIngredientUnit
+from mealie.schema.recipe.recipe_tool import RecipeToolSave
 from mealie.schema.response.pagination import PaginationQuery
 from mealie.services.seeder.seeder_service import SeederService
 from tests.utils import api_routes
@@ -172,6 +176,256 @@ def test_pagination_filter_basic(query_units: tuple[RepositoryUnit, IngredientUn
     assert unit_results[0].id == unit_2.id
 
 
+def test_pagination_filter_null(database: AllRepositories, unique_user: TestUser):
+    recipe_not_made_1 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=random_string())
+    )
+    recipe_not_made_2 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=random_string())
+    )
+
+    # give one recipe a last made date
+    recipe_made = database.recipes.create(
+        Recipe(
+            user_id=unique_user.user_id, group_id=unique_user.group_id, name=random_string(), last_made=datetime.now()
+        )
+    )
+
+    recipe_repo = database.recipes.by_group(unique_user.group_id)  # type: ignore
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter="lastMade IS NONE")
+    recipe_results = recipe_repo.page_all(query).items
+    assert len(recipe_results) == 2
+    result_ids = {result.id for result in recipe_results}
+    assert recipe_not_made_1.id in result_ids
+    assert recipe_not_made_2.id in result_ids
+    assert recipe_made.id not in result_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter="lastMade IS NULL")
+    recipe_results = recipe_repo.page_all(query).items
+    assert len(recipe_results) == 2
+    result_ids = {result.id for result in recipe_results}
+    assert recipe_not_made_1.id in result_ids
+    assert recipe_not_made_2.id in result_ids
+    assert recipe_made.id not in result_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter="lastMade IS NOT NONE")
+    recipe_results = recipe_repo.page_all(query).items
+    assert len(recipe_results) == 1
+    result_ids = {result.id for result in recipe_results}
+    assert recipe_not_made_1.id not in result_ids
+    assert recipe_not_made_2.id not in result_ids
+    assert recipe_made.id in result_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter="lastMade IS NOT NULL")
+    recipe_results = recipe_repo.page_all(query).items
+    assert len(recipe_results) == 1
+    result_ids = {result.id for result in recipe_results}
+    assert recipe_not_made_1.id not in result_ids
+    assert recipe_not_made_2.id not in result_ids
+    assert recipe_made.id in result_ids
+
+
+def test_pagination_filter_in(query_units: tuple[RepositoryUnit, IngredientUnit, IngredientUnit, IngredientUnit]):
+    units_repo, unit_1, unit_2, unit_3 = query_units
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter=f"name IN [{unit_1.name}, {unit_2.name}]")
+    unit_results = units_repo.page_all(query).items
+
+    assert len(unit_results) == 2
+    result_ids = {unit.id for unit in unit_results}
+    assert unit_1.id in result_ids
+    assert unit_2.id in result_ids
+    assert unit_3.id not in result_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter=f"name NOT IN [{unit_1.name}, {unit_2.name}]")
+    unit_results = units_repo.page_all(query).items
+
+    assert len(unit_results) == 1
+    result_ids = {unit.id for unit in unit_results}
+    assert unit_1.id not in result_ids
+    assert unit_2.id not in result_ids
+    assert unit_3.id in result_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter=f'name IN ["{unit_3.name}"]')
+    unit_results = units_repo.page_all(query).items
+
+    assert len(unit_results) == 1
+    result_ids = {unit.id for unit in unit_results}
+    assert unit_1.id not in result_ids
+    assert unit_2.id not in result_ids
+    assert unit_3.id in result_ids
+
+
+def test_pagination_filter_in_advanced(database: AllRepositories, unique_user: TestUser):
+    slug1, slug2 = (random_string(10) for _ in range(2))
+
+    tags = [
+        TagSave(group_id=unique_user.group_id, name=slug1, slug=slug1),
+        TagSave(group_id=unique_user.group_id, name=slug2, slug=slug2),
+    ]
+
+    tag_1, tag_2 = [database.tags.create(tag) for tag in tags]
+
+    # Bootstrap the database with recipes
+    slug = random_string()
+    recipe_0 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=slug, slug=slug, tags=[])
+    )
+
+    slug = random_string()
+    recipe_1 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=slug, slug=slug, tags=[tag_1])
+    )
+
+    slug = random_string()
+    recipe_2 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=slug, slug=slug, tags=[tag_2])
+    )
+
+    slug = random_string()
+    recipe_1_2 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=slug, slug=slug, tags=[tag_1, tag_2])
+    )
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter=f"tags.name IN [{tag_1.name}]")
+    recipe_results = database.recipes.page_all(query).items
+    assert len(recipe_results) == 2
+    recipe_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_0.id not in recipe_ids
+    assert recipe_1.id in recipe_ids
+    assert recipe_2.id not in recipe_ids
+    assert recipe_1_2.id in recipe_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter=f"tags.name IN [{tag_1.name}, {tag_2.name}]")
+    recipe_results = database.recipes.page_all(query).items
+    assert len(recipe_results) == 3
+    recipe_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_0.id not in recipe_ids
+    assert recipe_1.id in recipe_ids
+    assert recipe_2.id in recipe_ids
+    assert recipe_1_2.id in recipe_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter=f"tags.name CONTAINS ALL [{tag_1.name}, {tag_2.name}]")
+    recipe_results = database.recipes.page_all(query).items
+    assert len(recipe_results) == 1
+    recipe_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_0.id not in recipe_ids
+    assert recipe_1.id not in recipe_ids
+    assert recipe_2.id not in recipe_ids
+    assert recipe_1_2.id in recipe_ids
+
+
+def test_pagination_filter_like(query_units: tuple[RepositoryUnit, IngredientUnit, IngredientUnit, IngredientUnit]):
+    units_repo, unit_1, unit_2, unit_3 = query_units
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter=r'name LIKE "test u_it%"')
+    unit_results = units_repo.page_all(query).items
+
+    assert len(unit_results) == 3
+    result_ids = {unit.id for unit in unit_results}
+    assert unit_1.id in result_ids
+    assert unit_2.id in result_ids
+    assert unit_3.id in result_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter=r'name LIKE "%unit 1"')
+    unit_results = units_repo.page_all(query).items
+
+    assert len(unit_results) == 1
+    result_ids = {unit.id for unit in unit_results}
+    assert unit_1.id in result_ids
+    assert unit_2.id not in result_ids
+    assert unit_3.id not in result_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter=r'name NOT LIKE %t_1"')
+    unit_results = units_repo.page_all(query).items
+
+    assert len(unit_results) == 2
+    result_ids = {unit.id for unit in unit_results}
+    assert unit_1.id not in result_ids
+    assert unit_2.id in result_ids
+    assert unit_3.id in result_ids
+
+
+def test_pagination_filter_keyword_namespace_conflict(database: AllRepositories, unique_user: TestUser):
+    recipe_rating_1 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=random_string(), rating=1)
+    )
+    recipe_rating_2 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=random_string(), rating=2)
+    )
+
+    recipe_rating_3 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=random_string(), rating=3)
+    )
+
+    recipe_repo = database.recipes.by_group(unique_user.group_id)  # type: ignore
+
+    # "rating" contains the word "in", but we should not parse this as the keyword "IN"
+    query = PaginationQuery(page=1, per_page=-1, query_filter="rating > 2")
+    recipe_results = recipe_repo.page_all(query).items
+
+    assert len(recipe_results) == 1
+    result_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_rating_1.id not in result_ids
+    assert recipe_rating_2.id not in result_ids
+    assert recipe_rating_3.id in result_ids
+
+    query = PaginationQuery(page=1, per_page=-1, query_filter="rating in [1, 3]")
+    recipe_results = recipe_repo.page_all(query).items
+
+    assert len(recipe_results) == 2
+    result_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_rating_1.id in result_ids
+    assert recipe_rating_2.id not in result_ids
+    assert recipe_rating_3.id in result_ids
+
+
+def test_pagination_filter_logical_namespace_conflict(database: AllRepositories, unique_user: TestUser):
+    categories = [
+        CategorySave(group_id=unique_user.group_id, name=random_string(10)),
+        CategorySave(group_id=unique_user.group_id, name=random_string(10)),
+    ]
+    category_1, category_2 = [database.categories.create(category) for category in categories]
+
+    # Bootstrap the database with recipes
+    slug = random_string()
+    recipe_category_0 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=slug, slug=slug)
+    )
+
+    slug = random_string()
+    recipe_category_1 = database.recipes.create(
+        Recipe(
+            user_id=unique_user.user_id,
+            group_id=unique_user.group_id,
+            name=slug,
+            slug=slug,
+            recipe_category=[category_1],
+        )
+    )
+
+    slug = random_string()
+    recipe_category_2 = database.recipes.create(
+        Recipe(
+            user_id=unique_user.user_id,
+            group_id=unique_user.group_id,
+            name=slug,
+            slug=slug,
+            recipe_category=[category_2],
+        )
+    )
+
+    # "recipeCategory" has the substring "or" in it, which shouldn't break queries
+    query = PaginationQuery(page=1, per_page=-1, query_filter=f'recipeCategory.id = "{category_1.id}"')
+    recipe_results = database.recipes.by_group(unique_user.group_id).page_all(query).items  # type: ignore
+    assert len(recipe_results) == 1
+    recipe_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_category_0.id not in recipe_ids
+    assert recipe_category_1.id in recipe_ids
+    assert recipe_category_2.id not in recipe_ids
+
+
 def test_pagination_filter_datetimes(
     query_units: tuple[RepositoryUnit, IngredientUnit, IngredientUnit, IngredientUnit]
 ):
@@ -197,15 +451,183 @@ def test_pagination_filter_booleans(query_units: tuple[RepositoryUnit, Ingredien
 
 
 def test_pagination_filter_advanced(query_units: tuple[RepositoryUnit, IngredientUnit, IngredientUnit, IngredientUnit]):
-    units_repo = query_units[0]
-    unit_3 = query_units[3]
+    units_repo, unit_1, unit_2, unit_3 = query_units
 
     dt = str(unit_3.created_at.isoformat())  # type: ignore
-    qf = f'name="test unit 1" OR (useAbbreviation=f AND (name="test unit 2" OR createdAt > "{dt}"))'
+    qf = f'name="test unit 1" OR (useAbbreviation=f AND (name="{unit_2.name}" OR createdAt > "{dt}"))'
     query = PaginationQuery(page=1, per_page=-1, query_filter=qf)
     unit_results = units_repo.page_all(query).items
+
     assert len(unit_results) == 2
-    assert unit_3.id not in [unit.id for unit in unit_results]
+    result_ids = {unit.id for unit in unit_results}
+    assert unit_1.id in result_ids
+    assert unit_2.id in result_ids
+    assert unit_3.id not in result_ids
+
+    qf = f'(name LIKE %_1 OR name IN ["{unit_2.name}"]) AND createdAt IS NOT NONE'
+    query = PaginationQuery(page=1, per_page=-1, query_filter=qf)
+    unit_results = units_repo.page_all(query).items
+
+    assert len(unit_results) == 2
+    result_ids = {unit.id for unit in unit_results}
+    assert unit_1.id in result_ids
+    assert unit_2.id in result_ids
+    assert unit_3.id not in result_ids
+
+
+def test_pagination_filter_advanced_frontend_sort(database: AllRepositories, unique_user: TestUser):
+    categories = [
+        CategorySave(group_id=unique_user.group_id, name=random_string(10)),
+        CategorySave(group_id=unique_user.group_id, name=random_string(10)),
+    ]
+    category_1, category_2 = [database.categories.create(category) for category in categories]
+
+    slug1, slug2 = (random_string(10) for _ in range(2))
+    tags = [
+        TagSave(group_id=unique_user.group_id, name=slug1, slug=slug1),
+        TagSave(group_id=unique_user.group_id, name=slug2, slug=slug2),
+    ]
+    tag_1, tag_2 = [database.tags.create(tag) for tag in tags]
+
+    tools = [
+        RecipeToolSave(group_id=unique_user.group_id, name=random_string(10)),
+        RecipeToolSave(group_id=unique_user.group_id, name=random_string(10)),
+    ]
+    tool_1, tool_2 = [database.tools.create(tool) for tool in tools]
+
+    # Bootstrap the database with recipes
+    slug = random_string()
+    recipe_ct0_tg0_tl0 = database.recipes.create(
+        Recipe(user_id=unique_user.user_id, group_id=unique_user.group_id, name=slug, slug=slug)
+    )
+
+    slug = random_string()
+    recipe_ct1_tg0_tl0 = database.recipes.create(
+        Recipe(
+            user_id=unique_user.user_id,
+            group_id=unique_user.group_id,
+            name=slug,
+            slug=slug,
+            recipe_category=[category_1],
+        )
+    )
+
+    slug = random_string()
+    recipe_ct12_tg0_tl0 = database.recipes.create(
+        Recipe(
+            user_id=unique_user.user_id,
+            group_id=unique_user.group_id,
+            name=slug,
+            slug=slug,
+            recipe_category=[category_1, category_2],
+        )
+    )
+
+    slug = random_string()
+    recipe_ct1_tg1_tl0 = database.recipes.create(
+        Recipe(
+            user_id=unique_user.user_id,
+            group_id=unique_user.group_id,
+            name=slug,
+            slug=slug,
+            recipe_category=[category_1],
+            tags=[tag_1],
+        )
+    )
+
+    slug = random_string()
+    recipe_ct1_tg0_tl1 = database.recipes.create(
+        Recipe(
+            user_id=unique_user.user_id,
+            group_id=unique_user.group_id,
+            name=slug,
+            slug=slug,
+            recipe_category=[category_1],
+            tools=[tool_1],
+        )
+    )
+
+    slug = random_string()
+    recipe_ct0_tg2_tl2 = database.recipes.create(
+        Recipe(
+            user_id=unique_user.user_id,
+            group_id=unique_user.group_id,
+            name=slug,
+            slug=slug,
+            tags=[tag_2],
+            tools=[tool_2],
+        )
+    )
+
+    slug = random_string()
+    recipe_ct12_tg12_tl2 = database.recipes.create(
+        Recipe(
+            user_id=unique_user.user_id,
+            group_id=unique_user.group_id,
+            name=slug,
+            slug=slug,
+            recipe_category=[category_1, category_2],
+            tags=[tag_1, tag_2],
+            tools=[tool_2],
+        )
+    )
+
+    repo = database.recipes.by_group(unique_user.group_id)  # type: ignore
+
+    qf = f'recipeCategory.id IN ["{category_1.id}"] AND tools.id IN ["{tool_1.id}"]'
+    query = PaginationQuery(page=1, per_page=-1, query_filter=qf)
+    recipe_results = repo.page_all(query).items
+    assert len(recipe_results) == 1
+    recipe_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_ct0_tg0_tl0.id not in recipe_ids
+    assert recipe_ct1_tg0_tl0.id not in recipe_ids
+    assert recipe_ct12_tg0_tl0.id not in recipe_ids
+    assert recipe_ct1_tg1_tl0.id not in recipe_ids
+    assert recipe_ct1_tg0_tl1.id in recipe_ids
+    assert recipe_ct0_tg2_tl2.id not in recipe_ids
+    assert recipe_ct12_tg12_tl2.id not in recipe_ids
+
+    qf = f'recipeCategory.id CONTAINS ALL ["{category_1.id}", "{category_2.id}"] AND tags.id IN ["{tag_1.id}"]'
+    query = PaginationQuery(page=1, per_page=-1, query_filter=qf)
+    recipe_results = repo.page_all(query).items
+    assert len(recipe_results) == 1
+    recipe_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_ct0_tg0_tl0.id not in recipe_ids
+    assert recipe_ct1_tg0_tl0.id not in recipe_ids
+    assert recipe_ct12_tg0_tl0.id not in recipe_ids
+    assert recipe_ct1_tg1_tl0.id not in recipe_ids
+    assert recipe_ct1_tg0_tl1.id not in recipe_ids
+    assert recipe_ct0_tg2_tl2.id not in recipe_ids
+    assert recipe_ct12_tg12_tl2.id in recipe_ids
+
+    qf = f'tags.id IN ["{tag_1.id}", "{tag_2.id}"] AND tools.id IN ["{tool_2.id}"]'
+    query = PaginationQuery(page=1, per_page=-1, query_filter=qf)
+    recipe_results = repo.page_all(query).items
+    assert len(recipe_results) == 2
+    recipe_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_ct0_tg0_tl0.id not in recipe_ids
+    assert recipe_ct1_tg0_tl0.id not in recipe_ids
+    assert recipe_ct12_tg0_tl0.id not in recipe_ids
+    assert recipe_ct1_tg1_tl0.id not in recipe_ids
+    assert recipe_ct1_tg0_tl1.id not in recipe_ids
+    assert recipe_ct0_tg2_tl2.id in recipe_ids
+    assert recipe_ct12_tg12_tl2.id in recipe_ids
+
+    qf = (
+        f'recipeCategory.id CONTAINS ALL ["{category_1.id}", "{category_2.id}"]'
+        f'AND tags.id IN ["{tag_1.id}", "{tag_2.id}"] AND tools.id IN ["{tool_1.id}", "{tool_2.id}"]'
+    )
+    query = PaginationQuery(page=1, per_page=-1, query_filter=qf)
+    recipe_results = repo.page_all(query).items
+    assert len(recipe_results) == 1
+    recipe_ids = {recipe.id for recipe in recipe_results}
+    assert recipe_ct0_tg0_tl0.id not in recipe_ids
+    assert recipe_ct1_tg0_tl0.id not in recipe_ids
+    assert recipe_ct12_tg0_tl0.id not in recipe_ids
+    assert recipe_ct1_tg1_tl0.id not in recipe_ids
+    assert recipe_ct1_tg0_tl1.id not in recipe_ids
+    assert recipe_ct0_tg2_tl2.id not in recipe_ids
+    assert recipe_ct12_tg12_tl2.id in recipe_ids
 
 
 @pytest.mark.parametrize(
@@ -214,6 +636,13 @@ def test_pagination_filter_advanced(query_units: tuple[RepositoryUnit, Ingredien
         pytest.param('(name="test name" AND useAbbreviation=f))', id="unbalanced parenthesis"),
         pytest.param('id="this is not a valid UUID"', id="invalid UUID"),
         pytest.param('createdAt="this is not a valid datetime format"', id="invalid datetime format"),
+        pytest.param('name IS "test name"', id="IS can only be used with NULL or NONE"),
+        pytest.param('name IS NOT "test name"', id="IS NOT can only be used with NULL or NONE"),
+        pytest.param('name IN "test name"', id="IN must use a list of values"),
+        pytest.param('name NOT IN "test name"', id="NOT IN must use a list of values"),
+        pytest.param('name CONTAINS ALL "test name"', id="CONTAINS ALL must use a list of values"),
+        pytest.param('createdAt LIKE "2023-02-25"', id="LIKE is only valid for string columns"),
+        pytest.param('createdAt NOT LIKE "2023-02-25"', id="NOT LIKE is only valid for string columns"),
         pytest.param('badAttribute="test value"', id="invalid attribute"),
         pytest.param('group.badAttribute="test value"', id="bad nested attribute"),
         pytest.param('group.preferences.badAttribute="test value"', id="bad double nested attribute"),