From 3fbe75cef7fba9959d9a0a35d6d17aa4b7a0e42c Mon Sep 17 00:00:00 2001 From: Suren Khorenyan Date: Mon, 30 Oct 2023 23:18:21 +0300 Subject: [PATCH 1/2] inner jsonb filtering support (WIP) --- .../data_layers/filtering/sqlalchemy.py | 31 +++++- fastapi_jsonapi/utils/sqla.py | 19 +++- tests/common.py | 5 + tests/fixtures/db_connection.py | 1 + tests/models.py | 7 +- tests/test_api/test_api_sqla_with_includes.py | 2 - .../test_inner_schema_filtering/__init__.py | 0 .../test_filter_by_inner_json_schema.py | 101 ++++++++++++++++++ tests/test_atomic/test_create_objects.py | 3 - tests/test_atomic/test_delete_objects.py | 3 - tests/test_atomic/test_update_objects.py | 4 - 11 files changed, 159 insertions(+), 17 deletions(-) create mode 100644 tests/test_api/test_inner_schema_filtering/__init__.py create mode 100644 tests/test_api/test_inner_schema_filtering/test_filter_by_inner_json_schema.py diff --git a/fastapi_jsonapi/data_layers/filtering/sqlalchemy.py b/fastapi_jsonapi/data_layers/filtering/sqlalchemy.py index 0bdfc7aa..cc7ec192 100644 --- a/fastapi_jsonapi/data_layers/filtering/sqlalchemy.py +++ b/fastapi_jsonapi/data_layers/filtering/sqlalchemy.py @@ -6,6 +6,7 @@ from sqlalchemy import and_, not_, or_ from sqlalchemy.orm import InstrumentedAttribute, aliased from sqlalchemy.sql.elements import BinaryExpression +from sqlalchemy.sql.sqltypes import JSON from fastapi_jsonapi.data_layers.shared import create_filters_or_sorts from fastapi_jsonapi.data_typing import TypeModel, TypeSchema @@ -91,7 +92,8 @@ def create_filter(self, schema_field: ModelField, model_column, operator, value) raise InvalidType(detail=", ".join(errors)) return getattr(model_column, self.operator)(clear_value) - def resolve(self) -> FilterAndJoins: # noqa: PLR0911 + # TODO: refactor and remove ignore PLR0911, PLR0912 + def resolve(self) -> FilterAndJoins: # noqa: PLR0911, PLR0912 """Create filter for a particular node of the filter tree""" if "or" in self.filter_: return self._create_filters(type_filter="or") @@ -114,12 +116,23 @@ def resolve(self) -> FilterAndJoins: # noqa: PLR0911 operator=operator, ) + # TODO: check if relationship or inner schema + # TODO: create base schema `BaseJsonModel(BaseModel)`? reuse: + # https://github.com/AdCombo/combojsonapi/blob/45a43cf28c6496195c6e6762955db16f9a813b2f/combojsonapi/postgresql_jsonb/plugin.py#L103-L120 + + # todo: detect dynamically + IS_JSONB_FILTERING_EXACTLY_THIS_FIELD = True + if SPLIT_REL in self.filter_.get("name", ""): value = { "name": SPLIT_REL.join(self.filter_["name"].split(SPLIT_REL)[1:]), "op": operator, "val": value, } + + # TODO:!! + if IS_JSONB_FILTERING_EXACTLY_THIS_FIELD: + return self._json_inner_filtering(value) return self._relationship_filtering(value) if isinstance(value, dict): @@ -160,6 +173,13 @@ def _relationship_filtering(self, value): joins.extend(new_joins) return filters, joins + def _json_inner_filtering(self, value): + # TODO!! Upgrade Node usage :thinking: + node = Node(self.related_model, value, self.related_schema) + filters, new_joins = node.resolve() + # joins.extend(new_joins) + return filters, [] + def _create_filters(self, type_filter: str) -> FilterAndJoins: """ Создаём фильтр or или and @@ -215,6 +235,15 @@ def column(self) -> InstrumentedAttribute: model_field = get_model_field(self.schema, field) + # TODO!!! + # here + # if JSON: + # pass + + # ??? + if isinstance(self.model.type, JSON): + return self.model.op("->>")(model_field) + try: return getattr(self.model, model_field) except AttributeError: diff --git a/fastapi_jsonapi/utils/sqla.py b/fastapi_jsonapi/utils/sqla.py index 66b34360..8308715d 100644 --- a/fastapi_jsonapi/utils/sqla.py +++ b/fastapi_jsonapi/utils/sqla.py @@ -1,5 +1,11 @@ from typing import Type +# from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm.attributes import InstrumentedAttribute + +# from sqlalchemy import JSON +from sqlalchemy.sql.sqltypes import JSON + from fastapi_jsonapi.data_typing import TypeModel @@ -22,4 +28,15 @@ class ComputerSchema(pydantic_base): :param relation_name: :return: """ - return getattr(cls, relation_name).property.mapper.class_ + related_column: InstrumentedAttribute = getattr(cls, relation_name) + # TODO: any flags for JSON / JSONB? + # TODO: or any plugins to add support for JSON / JSONB, etc? + # TODO: https://github.com/AdCombo/combojsonapi/blob/45a43cf28c6496195c6e6762955db16f9a813b2f/combojsonapi/postgresql_jsonb/plugin.py#L103-L120 + # todo: check for json[b] + # if isinstance(related_column.type) in (JSON, ...): # TODO!! + if isinstance(related_column.type, JSON): + # return related_column.op("->>") + return related_column + + related_property = related_column.property + return related_property.mapper.class_ diff --git a/tests/common.py b/tests/common.py index fb0e3931..3a2f6ebb 100644 --- a/tests/common.py +++ b/tests/common.py @@ -8,3 +8,8 @@ def sqla_uri(): db_dir = Path(__file__).resolve().parent testing_db_url = f"sqlite+aiosqlite:///{db_dir}/db.sqlite3" return testing_db_url + + +db_uri = sqla_uri() +IS_POSTGRES = "postgres" in db_uri +IS_SQLITE = "sqlite" in db_uri diff --git a/tests/fixtures/db_connection.py b/tests/fixtures/db_connection.py index f0057467..30e61b5a 100644 --- a/tests/fixtures/db_connection.py +++ b/tests/fixtures/db_connection.py @@ -30,6 +30,7 @@ async def async_session_dependency(): async def async_engine(): engine = create_async_engine( url=make_url(sqla_uri()), + # TODO: env var echo=False, # echo=True, ) diff --git a/tests/models.py b/tests/models.py index 00a8bd16..102edf55 100644 --- a/tests/models.py +++ b/tests/models.py @@ -6,7 +6,7 @@ from sqlalchemy.orm import declared_attr, relationship from sqlalchemy.types import CHAR, TypeDecorator -from tests.common import sqla_uri +from tests.common import IS_POSTGRES, IS_SQLITE, sqla_uri class Base: @@ -74,6 +74,7 @@ class UserBio(AutoIdMixin, Base): birth_city: str = Column(String, nullable=False, default="", server_default="") favourite_movies: str = Column(String, nullable=False, default="", server_default="") keys_to_ids_list: Dict[str, List[int]] = Column(JSON) + meta: Dict[str, str] = Column(JSON, server_default="{}") user_id = Column(Integer, ForeignKey("users.id"), nullable=False, unique=True) user = relationship( @@ -267,10 +268,10 @@ def python_type(self): db_uri = sqla_uri() -if "postgres" in db_uri: +if IS_POSTGRES: # noinspection PyPep8Naming from sqlalchemy.dialects.postgresql import UUID as UUIDType -elif "sqlite" in db_uri: +elif IS_SQLITE: UUIDType = CustomUUIDType else: msg = "unsupported dialect (custom uuid?)" diff --git a/tests/test_api/test_api_sqla_with_includes.py b/tests/test_api/test_api_sqla_with_includes.py index b245132b..8205d987 100644 --- a/tests/test_api/test_api_sqla_with_includes.py +++ b/tests/test_api/test_api_sqla_with_includes.py @@ -39,8 +39,6 @@ pytestmark = mark.asyncio -logging.basicConfig(level=logging.DEBUG) - def association_key(data: dict): return data["type"], data["id"] diff --git a/tests/test_api/test_inner_schema_filtering/__init__.py b/tests/test_api/test_inner_schema_filtering/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_api/test_inner_schema_filtering/test_filter_by_inner_json_schema.py b/tests/test_api/test_inner_schema_filtering/test_filter_by_inner_json_schema.py new file mode 100644 index 00000000..6931b7b4 --- /dev/null +++ b/tests/test_api/test_inner_schema_filtering/test_filter_by_inner_json_schema.py @@ -0,0 +1,101 @@ +import pytest +import simplejson as json +from fastapi import FastAPI, status +from httpx import AsyncClient +from pydantic import BaseModel +from sqlalchemy.ext.asyncio import AsyncSession + +from tests.common import IS_POSTGRES +from tests.fixtures.app import build_app_custom +from tests.misc.utils import fake +from tests.models import ( + UserBio, +) +from tests.schemas import UserBioAttributesBaseSchema + +pytestmark = pytest.mark.asyncio + + +class UserBioMeta(BaseModel): + spam_and_eggs: str + + +class UserBioJsonMetaSchema(UserBioAttributesBaseSchema): + meta: UserBioMeta + + +@pytest.mark.skipif(not IS_POSTGRES, reason="only for pg (for now)") +class TestPostgresFilterByInnerSchema: + """ + Todo: + ---- + To create tests for fields: + - json + - jsonb + """ + + @pytest.fixture() + def resource_type(self) -> str: + return "user_bio_custom_json_meta" + + @pytest.fixture() + def app(self, resource_type): + app = build_app_custom( + model=UserBio, + schema=UserBioJsonMetaSchema, + resource_type=resource_type, + path=f"/{resource_type}", + ) + return app + + async def test_filter_inner_json_field( + self, + app: FastAPI, + resource_type: str, + client: AsyncClient, + async_session: AsyncSession, + user_1_bio: UserBio, + user_2_bio: UserBio, + ): + # Declared as UserBioMeta.spam_and_eggs + some_key = "spam_and_eggs" + # todo: use sentence and take part to check ilike using %{part}% + value_1 = fake.word() + value_2 = fake.word() + assert value_1 != value_2 + assert user_1_bio.id != user_2_bio.id + + await async_session.refresh(user_1_bio) + await async_session.refresh(user_2_bio) + + # re-assign meta dict! sqla doesn't watch mutations + user_1_bio.meta = {some_key: value_1} + user_2_bio.meta = {some_key: value_2} + await async_session.commit() + + filter_inner = { + "name": f"meta.{some_key}", + "op": "ilike", + "val": value_1, + } + query_params = { + "filter": json.dumps( + [ + filter_inner, + ], + ), + } + url = app.url_path_for(f"get_{resource_type}_list") + res = await client.get(url, params=query_params) + assert res.status_code == status.HTTP_200_OK, res.text + assert res.json() == { + "data": [ + { + "id": str(user_1_bio.id), + "type": resource_type, + "attributes": UserBioJsonMetaSchema.from_orm(user_1_bio).dict(), + }, + ], + "jsonapi": {"version": "1.0"}, + "meta": {"count": 1, "totalPages": 1}, + } diff --git a/tests/test_atomic/test_create_objects.py b/tests/test_atomic/test_create_objects.py index bea8f439..86c18b59 100644 --- a/tests/test_atomic/test_create_objects.py +++ b/tests/test_atomic/test_create_objects.py @@ -1,4 +1,3 @@ -import logging from typing import Callable import pytest @@ -26,8 +25,6 @@ pytestmark = mark.asyncio -logging.basicConfig(level=logging.DEBUG) - def random_sentence() -> str: return fake.sentence()[:COLUMN_CHARACTERS_LIMIT] diff --git a/tests/test_atomic/test_delete_objects.py b/tests/test_atomic/test_delete_objects.py index ac923b8d..d9ae549a 100644 --- a/tests/test_atomic/test_delete_objects.py +++ b/tests/test_atomic/test_delete_objects.py @@ -1,4 +1,3 @@ -import logging from typing import Awaitable, Callable from httpx import AsyncClient @@ -13,8 +12,6 @@ pytestmark = mark.asyncio -logging.basicConfig(level=logging.DEBUG) - class TestAtomicDeleteObjects: async def test_delete_two_objects( diff --git a/tests/test_atomic/test_update_objects.py b/tests/test_atomic/test_update_objects.py index 1074385c..71c9b517 100644 --- a/tests/test_atomic/test_update_objects.py +++ b/tests/test_atomic/test_update_objects.py @@ -1,5 +1,3 @@ -import logging - import pytest from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession @@ -11,8 +9,6 @@ pytestmark = pytest.mark.asyncio -logging.basicConfig(level=logging.DEBUG) - class TestAtomicUpdateObjects: async def test_update_two_objects( From 89c5276304270787e276d28a6fd3036234ac61e8 Mon Sep 17 00:00:00 2001 From: German Bernadskiy Date: Tue, 31 Oct 2023 14:48:19 +1000 Subject: [PATCH 2/2] fixes --- .../data_layers/filtering/sqlalchemy.py | 31 ++++++++++--------- fastapi_jsonapi/utils/sqla.py | 6 ++-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/fastapi_jsonapi/data_layers/filtering/sqlalchemy.py b/fastapi_jsonapi/data_layers/filtering/sqlalchemy.py index cc7ec192..2618dc48 100644 --- a/fastapi_jsonapi/data_layers/filtering/sqlalchemy.py +++ b/fastapi_jsonapi/data_layers/filtering/sqlalchemy.py @@ -1,11 +1,12 @@ """Helper to create sqlalchemy filters according to filter querystring parameter""" -from typing import Any, List, Tuple, Type, Union +from typing import Any, List, Optional, Tuple, Type, Union from pydantic import BaseModel from pydantic.fields import ModelField from sqlalchemy import and_, not_, or_ from sqlalchemy.orm import InstrumentedAttribute, aliased from sqlalchemy.sql.elements import BinaryExpression +from sqlalchemy.sql.schema import Column from sqlalchemy.sql.sqltypes import JSON from fastapi_jsonapi.data_layers.shared import create_filters_or_sorts @@ -92,6 +93,14 @@ def create_filter(self, schema_field: ModelField, model_column, operator, value) raise InvalidType(detail=", ".join(errors)) return getattr(model_column, self.operator)(clear_value) + def _is_json_column(self, column_name: str) -> bool: + column: Optional[Column] = self.model.__table__.columns.get(column_name) + + if column is None: + return False + + return isinstance(column.type, JSON) + # TODO: refactor and remove ignore PLR0911, PLR0912 def resolve(self) -> FilterAndJoins: # noqa: PLR0911, PLR0912 """Create filter for a particular node of the filter tree""" @@ -120,19 +129,18 @@ def resolve(self) -> FilterAndJoins: # noqa: PLR0911, PLR0912 # TODO: create base schema `BaseJsonModel(BaseModel)`? reuse: # https://github.com/AdCombo/combojsonapi/blob/45a43cf28c6496195c6e6762955db16f9a813b2f/combojsonapi/postgresql_jsonb/plugin.py#L103-L120 - # todo: detect dynamically - IS_JSONB_FILTERING_EXACTLY_THIS_FIELD = True - if SPLIT_REL in self.filter_.get("name", ""): + current_rel_or_column_name, *rel_names = self.filter_["name"].split(SPLIT_REL) value = { - "name": SPLIT_REL.join(self.filter_["name"].split(SPLIT_REL)[1:]), + "name": SPLIT_REL.join(rel_names), "op": operator, "val": value, } - # TODO:!! - if IS_JSONB_FILTERING_EXACTLY_THIS_FIELD: + is_json_filter = self._is_json_column(current_rel_or_column_name) + if is_json_filter: return self._json_inner_filtering(value) + return self._relationship_filtering(value) if isinstance(value, dict): @@ -235,13 +243,8 @@ def column(self) -> InstrumentedAttribute: model_field = get_model_field(self.schema, field) - # TODO!!! - # here - # if JSON: - # pass - - # ??? - if isinstance(self.model.type, JSON): + is_json_field = hasattr(self.model, "type") and isinstance(self.model.type, JSON) + if is_json_field: return self.model.op("->>")(model_field) try: diff --git a/fastapi_jsonapi/utils/sqla.py b/fastapi_jsonapi/utils/sqla.py index 8308715d..ed9581e9 100644 --- a/fastapi_jsonapi/utils/sqla.py +++ b/fastapi_jsonapi/utils/sqla.py @@ -32,9 +32,9 @@ class ComputerSchema(pydantic_base): # TODO: any flags for JSON / JSONB? # TODO: or any plugins to add support for JSON / JSONB, etc? # TODO: https://github.com/AdCombo/combojsonapi/blob/45a43cf28c6496195c6e6762955db16f9a813b2f/combojsonapi/postgresql_jsonb/plugin.py#L103-L120 - # todo: check for json[b] - # if isinstance(related_column.type) in (JSON, ...): # TODO!! - if isinstance(related_column.type, JSON): + + column_is_json = hasattr(related_column, "type") and isinstance(related_column.type, JSON) + if column_is_json: # return related_column.op("->>") return related_column