From 3919cfd094c6b2eea546c944c6541972944af15e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 23 Feb 2023 12:37:02 -0300 Subject: [PATCH 1/6] _asdict helper as classmethod --- itemadapter/adapter.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 9e015ab..4e91a73 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -361,17 +361,20 @@ def asdict(self) -> dict: """Return a dict object with the contents of the adapter. This works slightly different than calling `dict(adapter)`: it's applied recursively to nested items (if there are any). """ - return {key: _asdict(value) for key, value in self.items()} - - -def _asdict(obj: Any) -> Any: - """Helper for ItemAdapter.asdict().""" - if isinstance(obj, dict): - return {key: _asdict(value) for key, value in obj.items()} - if isinstance(obj, (list, set, tuple)): - return obj.__class__(_asdict(x) for x in obj) - if isinstance(obj, ItemAdapter): - return obj.asdict() - if ItemAdapter.is_item(obj): - return ItemAdapter(obj).asdict() - return obj + return {key: self._asdict(value) for key, value in self.items()} + + @classmethod + def _asdict(cls, obj: Any) -> Any: + if isinstance(obj, dict): + return {key: cls._asdict(value) for key, value in obj.items()} + if isinstance(obj, (list, set, tuple)): + return obj.__class__(cls._asdict(x) for x in obj) + if isinstance(obj, cls): + return obj.asdict() + if cls.is_item(obj): + return cls(obj).asdict() + if isinstance(obj, ItemAdapter): + return obj.asdict() + if ItemAdapter.is_item(obj): + return ItemAdapter(obj).asdict() + return obj From e881853271dd36e5e9209b5478ea91b18c1b67b7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 23 Feb 2023 13:24:48 -0300 Subject: [PATCH 2/6] pylint: ignore too-many-return-statements --- pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/pylintrc b/pylintrc index e89aaba..1fcbaa6 100644 --- a/pylintrc +++ b/pylintrc @@ -7,6 +7,7 @@ disable= missing-function-docstring, missing-module-docstring, raise-missing-from, + too-many-return-statements, unused-argument, From f6181df53121baa2dfcc9047e42bae6d141644d7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 23 Feb 2023 13:25:31 -0300 Subject: [PATCH 3/6] Readme: example about ItemAdapter subclassing --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/README.md b/README.md index 61b30d3..ecd6826 100644 --- a/README.md +++ b/README.md @@ -374,6 +374,48 @@ class attribute in order to handle custom item classes: >>> ``` +### Multiple adapter classes + +If you need to have different handlers and/or priorities for different cases +you can subclass the `ItemAdapter` class and set the `ADAPTER_CLASSES` +attribute as needed: + + +**Example** +```python +from collections import deque + +from itemadapter.adapter import ( + ItemAdapter, + AttrsAdapter, + DataclassAdapter, + DictAdapter, + PydanticAdapter, + ScrapyItemAdapter, +) +from scrapy.item import Item, Field + + +class BuiltinTypesItemAdapter(ItemAdapter): + ADAPTER_CLASSES = deque([DictAdapter, DataclassAdapter]) + +class ThirdPartyTypesItemAdapter(ItemAdapter): + ADAPTER_CLASSES = deque([AttrsAdapter, PydanticAdapter, ScrapyItemAdapter]) + +class ScrapyItem(Item): + foo = Field() +``` +```python +>>> BuiltinTypesItemAdapter.is_item(dict()) +True +>>> ThirdPartyTypesItemAdapter.is_item(dict()) +False +>>> BuiltinTypesItemAdapter.is_item(ScrapyItem(foo="bar")) +False +>>> ThirdPartyTypesItemAdapter.is_item(ScrapyItem(foo="bar")) +True +``` + --- ## More examples From 5c6a984b6abd42cbe235bd002f44de688d962ab4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 23 Feb 2023 14:16:02 -0300 Subject: [PATCH 4/6] Fix code snippet in Readme --- README.md | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index ecd6826..ae32b0a 100644 --- a/README.md +++ b/README.md @@ -383,29 +383,26 @@ attribute as needed: **Example** ```python -from collections import deque - -from itemadapter.adapter import ( - ItemAdapter, - AttrsAdapter, - DataclassAdapter, - DictAdapter, - PydanticAdapter, - ScrapyItemAdapter, -) -from scrapy.item import Item, Field - - -class BuiltinTypesItemAdapter(ItemAdapter): - ADAPTER_CLASSES = deque([DictAdapter, DataclassAdapter]) - -class ThirdPartyTypesItemAdapter(ItemAdapter): - ADAPTER_CLASSES = deque([AttrsAdapter, PydanticAdapter, ScrapyItemAdapter]) - -class ScrapyItem(Item): - foo = Field() -``` -```python +>>> from collections import deque +>>> from itemadapter.adapter import ( +... ItemAdapter, +... AttrsAdapter, +... DataclassAdapter, +... DictAdapter, +... PydanticAdapter, +... ScrapyItemAdapter, +... ) +>>> from scrapy.item import Item, Field +>>> +>>> class BuiltinTypesItemAdapter(ItemAdapter): +... ADAPTER_CLASSES = deque([DictAdapter, DataclassAdapter]) +... +>>> class ThirdPartyTypesItemAdapter(ItemAdapter): +... ADAPTER_CLASSES = deque([AttrsAdapter, PydanticAdapter, ScrapyItemAdapter]) +... +>>> class ScrapyItem(Item): +... foo = Field() +... >>> BuiltinTypesItemAdapter.is_item(dict()) True >>> ThirdPartyTypesItemAdapter.is_item(dict()) @@ -414,6 +411,7 @@ False False >>> ThirdPartyTypesItemAdapter.is_item(ScrapyItem(foo="bar")) True +>>> ``` --- From 2732297ec28ee768b03d3ed27bf410d18649284f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 26 Feb 2023 13:34:13 -0300 Subject: [PATCH 5/6] Add tests --- tests/test_itemadapter.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/test_itemadapter.py b/tests/test_itemadapter.py index 1acaaad..f36a101 100644 --- a/tests/test_itemadapter.py +++ b/tests/test_itemadapter.py @@ -1,10 +1,13 @@ import unittest +from collections import deque -from itemadapter.adapter import ItemAdapter +from itemadapter.adapter import ItemAdapter, DictAdapter +from tests import DataClassItem -class SubclassedItemAdapter(ItemAdapter): - pass + +class DictOnlyItemAdapter(ItemAdapter): + ADAPTER_CLASSES = deque([DictAdapter]) class ItemAdapterTestCase(unittest.TestCase): @@ -13,5 +16,23 @@ def test_repr(self): self.assertEqual(repr(adapter), "") def test_repr_subclass(self): - adapter = SubclassedItemAdapter(dict(foo="bar")) - self.assertEqual(repr(adapter), "") + adapter = DictOnlyItemAdapter(dict(foo="bar")) + self.assertEqual(repr(adapter), "") + + def test_as_dict_subclass(self): + """'asdict' method of ItemAdapter subclasses handles items even + if they're not handled by the subclass itself. + """ + item = dict( + foo="bar", + dataclass_item=DataClassItem(name="asdf", value="qwerty"), + itemadapter=ItemAdapter({"a": 1, "b": 2}), + ) + self.assertEqual( + DictOnlyItemAdapter(item).asdict(), + dict( + foo="bar", + dataclass_item=dict(name="asdf", value="qwerty"), + itemadapter={"a": 1, "b": 2}, + ), + ) From 5882f43a59563d1747859c32152c7127d34bfc4e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 28 Mar 2023 10:05:28 -0300 Subject: [PATCH 6/6] Simplify _asdict helper --- itemadapter/adapter.py | 4 ---- tests/test_itemadapter.py | 20 -------------------- 2 files changed, 24 deletions(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 4e91a73..5b36353 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -373,8 +373,4 @@ def _asdict(cls, obj: Any) -> Any: return obj.asdict() if cls.is_item(obj): return cls(obj).asdict() - if isinstance(obj, ItemAdapter): - return obj.asdict() - if ItemAdapter.is_item(obj): - return ItemAdapter(obj).asdict() return obj diff --git a/tests/test_itemadapter.py b/tests/test_itemadapter.py index f36a101..91f7a21 100644 --- a/tests/test_itemadapter.py +++ b/tests/test_itemadapter.py @@ -3,8 +3,6 @@ from itemadapter.adapter import ItemAdapter, DictAdapter -from tests import DataClassItem - class DictOnlyItemAdapter(ItemAdapter): ADAPTER_CLASSES = deque([DictAdapter]) @@ -18,21 +16,3 @@ def test_repr(self): def test_repr_subclass(self): adapter = DictOnlyItemAdapter(dict(foo="bar")) self.assertEqual(repr(adapter), "") - - def test_as_dict_subclass(self): - """'asdict' method of ItemAdapter subclasses handles items even - if they're not handled by the subclass itself. - """ - item = dict( - foo="bar", - dataclass_item=DataClassItem(name="asdf", value="qwerty"), - itemadapter=ItemAdapter({"a": 1, "b": 2}), - ) - self.assertEqual( - DictOnlyItemAdapter(item).asdict(), - dict( - foo="bar", - dataclass_item=dict(name="asdf", value="qwerty"), - itemadapter={"a": 1, "b": 2}, - ), - )