From f81d9af418308073e8a6f0a5bfa635e0a5ac43ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Thu, 7 Nov 2024 23:59:36 +0100 Subject: [PATCH] Tin/defaultdicts (#588) * Defaultdicts WIP * Reformat * Docs * More docs * Tweak docs * Introduce SimpleStructureHook --- HISTORY.md | 12 +- docs/customizing.md | 241 +++++++++++++++++++------------------ docs/defaulthooks.md | 40 ++++++ src/cattrs/__init__.py | 28 +++-- src/cattrs/cols.py | 48 +++++++- src/cattrs/converters.py | 16 ++- src/cattrs/gen/__init__.py | 13 +- src/cattrs/types.py | 12 ++ tests/test_defaultdicts.py | 32 +++++ 9 files changed, 299 insertions(+), 143 deletions(-) create mode 100644 src/cattrs/types.py create mode 100644 tests/test_defaultdicts.py diff --git a/HISTORY.md b/HISTORY.md index 8bec3ee8..64f4425d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -14,15 +14,21 @@ Our backwards-compatibility policy can be found [here](https://github.com/python - **Potentially breaking**: The converters raise {class}`StructureHandlerNotFoundError` more eagerly (on hook creation, instead of on hook use). This helps surfacing problems with missing hooks sooner. - See [Migrations](https://catt.rs/latest/migrations.html#the-default-structure-hook-fallback-factory) for steps to restore legacy behavior. + See [Migrations](https://catt.rs/en/latest/migrations.html#the-default-structure-hook-fallback-factory) for steps to restore legacy behavior. ([#577](https://github.com/python-attrs/cattrs/pull/577)) -- Add a [Migrations](https://catt.rs/latest/migrations.html) page, with instructions on migrating changed behavior for each version. +- Add a [Migrations](https://catt.rs/en/latest/migrations.html) page, with instructions on migrating changed behavior for each version. ([#577](https://github.com/python-attrs/cattrs/pull/577)) - Expose {func}`cattrs.cols.mapping_unstructure_factory` through {mod}`cattrs.cols`. +- Some `defaultdicts` are now [supported by default](https://catt.rs/en/latest/defaulthooks.html#defaultdicts), and + {func}`cattrs.cols.is_defaultdict`{func} and `cattrs.cols.defaultdict_structure_factory` are exposed through {mod}`cattrs.cols`. + ([#519](https://github.com/python-attrs/cattrs/issues/519) [#588](https://github.com/python-attrs/cattrs/pull/588)) +- Replace `cattrs.gen.MappingStructureFn` with `cattrs.SimpleStructureHook[In, T]`. - Python 3.13 is now supported. ([#543](https://github.com/python-attrs/cattrs/pull/543) [#547](https://github.com/python-attrs/cattrs/issues/547)) - Python 3.8 is no longer supported, as it is end-of-life. Use previous versions on this Python version. -- Change type of Converter.__init__.unstruct_collection_overrides from Callable to Mapping[type, UnstructureHook] ([#594](https://github.com/python-attrs/cattrs/pull/594). + ([#591](https://github.com/python-attrs/cattrs/pull/591)) +- Change type of `Converter.__init__.unstruct_collection_overrides` from `Callable` to `Mapping[type, UnstructureHook]` + ([#594](https://github.com/python-attrs/cattrs/pull/594)). ## 24.1.2 (2024-09-22) diff --git a/docs/customizing.md b/docs/customizing.md index c8860e04..ef46c5d7 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -155,131 +155,20 @@ Here's an example of using an unstructure hook factory to handle unstructuring [ [1, 2] ``` -## Customizing Collections - -The {mod}`cattrs.cols` module contains predicates and hook factories useful for customizing collection handling. -These hook factories can be wrapped to apply complex customizations. - -Available predicates are: - -* {meth}`is_any_set ` -* {meth}`is_frozenset ` -* {meth}`is_set ` -* {meth}`is_sequence ` -* {meth}`is_mapping ` -* {meth}`is_namedtuple ` - -````{tip} -These predicates aren't _cattrs_-specific and may be useful in other contexts. -```{doctest} predicates ->>> from cattrs.cols import is_sequence - ->>> is_sequence(list[str]) -True -``` -```` - - -Available hook factories are: - -* {meth}`iterable_unstructure_factory ` -* {meth}`list_structure_factory ` -* {meth}`namedtuple_structure_factory ` -* {meth}`namedtuple_unstructure_factory ` -* {meth}`namedtuple_dict_structure_factory ` -* {meth}`namedtuple_dict_unstructure_factory ` -* {meth}`mapping_structure_factory ` -* {meth}`mapping_unstructure_factory ` - -Additional predicates and hook factories will be added as requested. - -For example, by default sequences are structured from any iterable into lists. -This may be too lax, and additional validation may be applied by wrapping the default list structuring hook factory. - -```{testcode} list-customization -from cattrs.cols import is_sequence, list_structure_factory - -c = Converter() - -@c.register_structure_hook_factory(is_sequence) -def strict_list_hook_factory(type, converter): - - # First, we generate the default hook... - list_hook = list_structure_factory(type, converter) - - # Then, we wrap it with a function of our own... - def strict_list_hook(value, type): - if not isinstance(value, list): - raise ValueError("Not a list!") - return list_hook(value, type) +## Using `cattrs.gen` Hook Factories - # And finally, we return our own composite hook. - return strict_list_hook -``` - -Now, all sequence structuring will be stricter: - -```{doctest} list-customization ->>> c.structure({"a", "b", "c"}, list[str]) -Traceback (most recent call last): - ... -ValueError: Not a list! -``` - -```{versionadded} 24.1.0 - -``` - -### Customizing Named Tuples - -Named tuples can be un/structured using dictionaries using the {meth}`namedtuple_dict_structure_factory ` -and {meth}`namedtuple_dict_unstructure_factory ` -hook factories. - -To unstructure _all_ named tuples into dictionaries: - -```{doctest} namedtuples ->>> from typing import NamedTuple - ->>> from cattrs.cols import is_namedtuple, namedtuple_dict_unstructure_factory ->>> c = Converter() - ->>> c.register_unstructure_hook_factory(is_namedtuple, namedtuple_dict_unstructure_factory) - - ->>> class MyNamedTuple(NamedTuple): -... a: int - ->>> c.unstructure(MyNamedTuple(1)) -{'a': 1} -``` - -To only un/structure _some_ named tuples into dictionaries, -change the predicate function when registering the hook factory: - -```{doctest} namedtuples - :options: +ELLIPSIS - ->>> c.register_unstructure_hook_factory( -... lambda t: t is MyNamedTuple, -... namedtuple_dict_unstructure_factory, -... ) - -``` - -## Using `cattrs.gen` Generators - -The {mod}`cattrs.gen` module allows for generating and compiling specialized hooks for unstructuring _attrs_ classes, dataclasses and typed dicts. +The {mod}`cattrs.gen` module contains [hook factories](#hook-factories) for un/structuring _attrs_ classes, dataclasses and typed dicts. The default {class}`Converter `, upon first encountering one of these types, -will use the generation functions mentioned here to generate specialized hooks for it, +will use the hook factories mentioned here to generate specialized hooks for it, register the hooks and use them. One reason for generating these hooks in advance is that they can bypass a lot of _cattrs_ machinery and be significantly faster than normal _cattrs_. -The hooks are also good building blocks for more complex customizations. +The hook factories are also good building blocks for more complex customizations. Another reason is overriding behavior on a per-attribute basis. -Currently, the overrides only support generating dictionary un/structuring hooks (as opposed to tuples), and support `omit_if_default`, `forbid_extra_keys`, `rename` and `omit`. +Currently, the overrides only support generating dictionary un/structuring hooks (as opposed to tuples), +and support `omit_if_default`, `forbid_extra_keys`, `rename` and `omit`. ### `omit_if_default` @@ -491,3 +380,121 @@ ClassWithInitFalse(number=2) ```{versionadded} 23.2.0 ``` + +## Customizing Collections + +The {mod}`cattrs.cols` module contains predicates and hook factories useful for customizing collection handling. +These hook factories can be wrapped to apply complex customizations. + +Available predicates are: + +* {meth}`is_any_set ` +* {meth}`is_frozenset ` +* {meth}`is_set ` +* {meth}`is_sequence ` +* {meth}`is_mapping ` +* {meth}`is_namedtuple ` +* {meth}`is_defaultdict ` + +````{tip} +These predicates aren't _cattrs_-specific and may be useful in other contexts. +```{doctest} predicates +>>> from cattrs.cols import is_sequence + +>>> is_sequence(list[str]) +True +``` +```` + + +Available hook factories are: + +* {meth}`iterable_unstructure_factory ` +* {meth}`list_structure_factory ` +* {meth}`namedtuple_structure_factory ` +* {meth}`namedtuple_unstructure_factory ` +* {meth}`namedtuple_dict_structure_factory ` +* {meth}`namedtuple_dict_unstructure_factory ` +* {meth}`mapping_structure_factory ` +* {meth}`mapping_unstructure_factory ` +* {meth}`defaultdict_structure_factory ` + +Additional predicates and hook factories will be added as requested. + +For example, by default sequences are structured from any iterable into lists. +This may be too lax, and additional validation may be applied by wrapping the default list structuring hook factory. + +```{testcode} list-customization +from cattrs.cols import is_sequence, list_structure_factory + +c = Converter() + +@c.register_structure_hook_factory(is_sequence) +def strict_list_hook_factory(type, converter): + + # First, we generate the default hook... + list_hook = list_structure_factory(type, converter) + + # Then, we wrap it with a function of our own... + def strict_list_hook(value, type): + if not isinstance(value, list): + raise ValueError("Not a list!") + return list_hook(value, type) + + # And finally, we return our own composite hook. + return strict_list_hook +``` + +Now, all sequence structuring will be stricter: + +```{doctest} list-customization +>>> c.structure({"a", "b", "c"}, list[str]) +Traceback (most recent call last): + ... +ValueError: Not a list! +``` + +```{versionadded} 24.1.0 + +``` + +### Customizing Named Tuples + +Named tuples can be un/structured using dictionaries using the {meth}`namedtuple_dict_structure_factory ` +and {meth}`namedtuple_dict_unstructure_factory ` +hook factories. + +To unstructure _all_ named tuples into dictionaries: + +```{doctest} namedtuples +>>> from typing import NamedTuple + +>>> from cattrs.cols import is_namedtuple, namedtuple_dict_unstructure_factory +>>> c = Converter() + +>>> c.register_unstructure_hook_factory(is_namedtuple, namedtuple_dict_unstructure_factory) + + +>>> class MyNamedTuple(NamedTuple): +... a: int + +>>> c.unstructure(MyNamedTuple(1)) +{'a': 1} +``` + +To only un/structure _some_ named tuples into dictionaries, +change the predicate function when registering the hook factory: + +```{doctest} namedtuples + :options: +ELLIPSIS + +>>> c.register_unstructure_hook_factory( +... lambda t: t is MyNamedTuple, +... namedtuple_dict_unstructure_factory, +... ) + +``` + +```{versionadded} 24.1.0 + +``` \ No newline at end of file diff --git a/docs/defaulthooks.md b/docs/defaulthooks.md index fb819555..3ae4f23c 100644 --- a/docs/defaulthooks.md +++ b/docs/defaulthooks.md @@ -183,6 +183,46 @@ Both keys and values are converted. {'1': None, '2': 2} ``` +### defaultdicts + +[`defaultdicts`](https://docs.python.org/3/library/collections.html#collections.defaultdict) +can be structured by default if they can be initialized using their value type hint. +Supported types are: + +- `collections.defaultdict[K, V]` +- `typing.DefaultDict[K, V]` + +For example, `defaultdict[str, int]` works since _cattrs_ will initialize it with `defaultdict(int)`. + +This also means `defaultdicts` without key and value annotations (bare `defaultdicts`) cannot be structured by default. + +`defaultdicts` with arbitrary default factories can be structured by using {meth}`defaultdict_structure_factory `: + +```{doctest} +>>> from collections import defaultdict +>>> from cattrs.cols import defaultdict_structure_factory + +>>> converter = Converter() +>>> hook = defaultdict_structure_factory( +... defaultdict[str, int], +... converter, +... default_factory=lambda: 1 +... ) + +>>> hook({"key": 1}) +defaultdict( at ...>, {'key': 1}) +``` + +`defaultdicts` are unstructured into plain dictionaries. + +```{note} +`defaultdicts` are not supported by the BaseConverter. +``` + +```{versionadded} 24.2.0 + +``` + ### Virtual Subclasses of [`abc.Mapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.Mapping) and [`abc.MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping) If a class declares itself a virtual subclass of `collections.abc.Mapping` or `collections.abc.MutableMapping` and its initializer accepts a dictionary, diff --git a/src/cattrs/__init__.py b/src/cattrs/__init__.py index db496363..18ab4aea 100644 --- a/src/cattrs/__init__.py +++ b/src/cattrs/__init__.py @@ -11,32 +11,34 @@ StructureHandlerNotFoundError, ) from .gen import override +from .types import SimpleStructureHook from .v import transform_error __all__ = [ - "structure", - "unstructure", - "get_structure_hook", - "get_unstructure_hook", - "register_structure_hook_func", - "register_structure_hook", - "register_unstructure_hook_func", - "register_unstructure_hook", - "structure_attrs_fromdict", - "structure_attrs_fromtuple", - "global_converter", - "BaseConverter", - "Converter", "AttributeValidationNote", + "BaseConverter", "BaseValidationError", "ClassValidationError", + "Converter", "ForbiddenExtraKeysError", "GenConverter", + "get_structure_hook", + "get_unstructure_hook", + "global_converter", "IterableValidationError", "IterableValidationNote", "override", + "register_structure_hook_func", + "register_structure_hook", + "register_unstructure_hook_func", + "register_unstructure_hook", + "SimpleStructureHook", + "structure_attrs_fromdict", + "structure_attrs_fromtuple", + "structure", "StructureHandlerNotFoundError", "transform_error", + "unstructure", "UnstructureStrategy", ] diff --git a/src/cattrs/cols.py b/src/cattrs/cols.py index 43d225f8..fc2ac986 100644 --- a/src/cattrs/cols.py +++ b/src/cattrs/cols.py @@ -2,12 +2,31 @@ from __future__ import annotations +from collections import defaultdict from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, Literal, NamedTuple, TypeVar, get_type_hints +from functools import partial +from typing import ( + TYPE_CHECKING, + Any, + DefaultDict, + Literal, + NamedTuple, + TypeVar, + get_type_hints, +) from attrs import NOTHING, Attribute -from ._compat import ANIES, is_bare, is_frozenset, is_mapping, is_sequence, is_subclass +from ._compat import ( + ANIES, + get_args, + get_origin, + is_bare, + is_frozenset, + is_mapping, + is_sequence, + is_subclass, +) from ._compat import is_mutable_set as is_set from .dispatch import StructureHook, UnstructureHook from .errors import IterableValidationError, IterableValidationNote @@ -28,11 +47,13 @@ __all__ = [ "is_any_set", + "is_defaultdict", "is_frozenset", "is_namedtuple", "is_mapping", "is_set", "is_sequence", + "defaultdict_structure_factory", "iterable_unstructure_factory", "list_structure_factory", "namedtuple_structure_factory", @@ -261,3 +282,26 @@ def namedtuple_dict_unstructure_factory( working_set.remove(cl) if not working_set: del already_generating.working_set + + +def is_defaultdict(type: Any) -> bool: + """Is this type a defaultdict? + + Bare defaultdicts (defaultdicts with no type arguments) are not supported + since there's no way to discover their _default_factory_. + """ + return is_subclass(get_origin(type), (defaultdict, DefaultDict)) + + +def defaultdict_structure_factory( + type: type[defaultdict], converter: BaseConverter, default_factory: Any = NOTHING +) -> StructureHook: + """A structure hook factory for defaultdicts. + + The value type parameter will be used as the _default factory_. + """ + if default_factory is NOTHING: + default_factory = get_args(type)[1] + return mapping_structure_factory( + type, converter, partial(defaultdict, default_factory) + ) diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index c21f5ea7..0f8a7bb2 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -54,6 +54,8 @@ signature, ) from .cols import ( + defaultdict_structure_factory, + is_defaultdict, is_namedtuple, iterable_unstructure_factory, list_structure_factory, @@ -83,7 +85,6 @@ DictStructureFn, HeteroTupleUnstructureFn, IterableUnstructureFn, - MappingStructureFn, MappingUnstructureFn, make_dict_structure_fn, make_dict_unstructure_fn, @@ -91,6 +92,7 @@ ) from .gen.typeddicts import make_dict_structure_fn as make_typeddict_dict_struct_fn from .gen.typeddicts import make_dict_unstructure_fn as make_typeddict_dict_unstruct_fn +from .types import SimpleStructureHook __all__ = ["UnstructureStrategy", "BaseConverter", "Converter", "GenConverter"] @@ -135,6 +137,7 @@ UnstructureHookT = TypeVar("UnstructureHookT", bound=UnstructureHook) StructureHookT = TypeVar("StructureHookT", bound=StructureHook) +CounterT = TypeVar("CounterT", bound=Counter) class UnstructureStrategy(Enum): @@ -1170,6 +1173,9 @@ def __init__( self.register_structure_hook_factory(is_annotated, self.gen_structure_annotated) self.register_structure_hook_factory(is_mapping, self.gen_structure_mapping) self.register_structure_hook_factory(is_counter, self.gen_structure_counter) + self.register_structure_hook_factory( + is_defaultdict, defaultdict_structure_factory + ) self.register_structure_hook_factory(is_typeddict, self.gen_structure_typeddict) self.register_structure_hook_factory( lambda t: get_newtype_base(t) is not None, self.get_structure_newtype @@ -1337,7 +1343,9 @@ def gen_unstructure_mapping( self._unstructure_func.register_cls_list([(cl, h)], direct=True) return h - def gen_structure_counter(self, cl: Any) -> MappingStructureFn[T]: + def gen_structure_counter( + self, cl: type[CounterT] + ) -> SimpleStructureHook[Mapping[Any, Any], CounterT]: h = mapping_structure_factory( cl, self, @@ -1348,7 +1356,9 @@ def gen_structure_counter(self, cl: Any) -> MappingStructureFn[T]: self._structure_func.register_cls_list([(cl, h)], direct=True) return h - def gen_structure_mapping(self, cl: Any) -> MappingStructureFn[T]: + def gen_structure_mapping( + self, cl: Any + ) -> SimpleStructureHook[Mapping[Any, Any], Any]: structure_to = get_origin(cl) or cl if structure_to in ( MutableMapping, diff --git a/src/cattrs/gen/__init__.py b/src/cattrs/gen/__init__.py index 9b1c3793..25e7f431 100644 --- a/src/cattrs/gen/__init__.py +++ b/src/cattrs/gen/__init__.py @@ -29,6 +29,7 @@ StructureHandlerNotFoundError, ) from ..fns import identity +from ..types import SimpleStructureHook from ._consts import AttributeOverride, already_generating, neutral from ._generics import generate_mapping from ._lc import generate_unique_filename @@ -892,8 +893,6 @@ def mapping_unstructure_factory( make_mapping_unstructure_fn: Final = mapping_unstructure_factory -MappingStructureFn = Callable[[Mapping[Any, Any], Any], T] - # This factory is here for backwards compatibility and circular imports. def mapping_structure_factory( @@ -902,11 +901,14 @@ def mapping_structure_factory( structure_to: type = dict, key_type=NOTHING, val_type=NOTHING, - detailed_validation: bool = True, -) -> MappingStructureFn[T]: + detailed_validation: bool | Literal["from_converter"] = "from_converter", +) -> SimpleStructureHook[Mapping[Any, Any], T]: """Generate a specialized structure function for a mapping.""" fn_name = "structure_mapping" + if detailed_validation == "from_converter": + detailed_validation = converter.detailed_validation + globs: dict[str, type] = {"__cattr_mapping_cl": structure_to} lines = [] @@ -1007,7 +1009,8 @@ def mapping_structure_factory( for k, v in internal_arg_parts.items(): globs[k] = v - def_line = f"def {fn_name}(mapping, _{internal_arg_line}):" + globs["cl"] = cl + def_line = f"def {fn_name}(mapping, cl=cl{internal_arg_line}):" total_lines = [def_line, *lines, " return res"] script = "\n".join(total_lines) diff --git a/src/cattrs/types.py b/src/cattrs/types.py new file mode 100644 index 00000000..a864cb90 --- /dev/null +++ b/src/cattrs/types.py @@ -0,0 +1,12 @@ +from typing import Protocol, TypeVar + +__all__ = ["SimpleStructureHook"] + +In = TypeVar("In") +T = TypeVar("T") + + +class SimpleStructureHook(Protocol[In, T]): + """A structure hook with an optional (ignored) second argument.""" + + def __call__(self, _: In, /, cl=...) -> T: ... diff --git a/tests/test_defaultdicts.py b/tests/test_defaultdicts.py new file mode 100644 index 00000000..02a34637 --- /dev/null +++ b/tests/test_defaultdicts.py @@ -0,0 +1,32 @@ +"""Tests for defaultdicts.""" + +from collections import defaultdict +from typing import DefaultDict + +from cattrs import Converter + + +def test_typing_defaultdicts(genconverter: Converter): + """`typing.DefaultDict` works.""" + res = genconverter.structure({"a": 1}, DefaultDict[str, int]) + + assert isinstance(res, defaultdict) + assert res["a"] == 1 + assert res["b"] == 0 + + genconverter.register_unstructure_hook(int, str) + + assert genconverter.unstructure(res) == {"a": "1", "b": "0"} + + +def test_collection_defaultdicts(genconverter: Converter): + """`collections.defaultdict` works.""" + res = genconverter.structure({"a": 1}, defaultdict[str, int]) + + assert isinstance(res, defaultdict) + assert res["a"] == 1 + assert res["b"] == 0 + + genconverter.register_unstructure_hook(int, str) + + assert genconverter.unstructure(res) == {"a": "1", "b": "0"}