From 921e8e4b07b5b2b8fcccd5c6d049963239d40bc9 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 11 Mar 2020 23:29:55 -0400 Subject: [PATCH 01/53] first attempt at an adjacently tagged union for consideration --- src/desert/_fields.py | 84 +++++++++++++++++++++++++++++++++++++++++++ tests/test_fields.py | 74 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 src/desert/_fields.py create mode 100644 tests/test_fields.py diff --git a/src/desert/_fields.py b/src/desert/_fields.py new file mode 100644 index 0000000..1272d37 --- /dev/null +++ b/src/desert/_fields.py @@ -0,0 +1,84 @@ +import typing + +import attr +import marshmallow.fields + + +T = typing.TypeVar('T') + + +@attr.s(frozen=True, auto_attribs=True) +class TypeTagField: + cls: type + tag: str + field: marshmallow.fields.Field + + +@attr.s(auto_attribs=True) +class TypeDictRegistry: + the_dict: typing.Dict[ + typing.Union[type, str], + marshmallow.fields.Field, + ] = attr.ib(factory=dict) + + def register(self, cls, tag, field): + if any(key in self.the_dict for key in [cls, tag]): + raise Exception() + + type_tag_field = TypeTagField(cls=cls, tag=tag, field=field) + + self.the_dict[cls] = type_tag_field + self.the_dict[tag] = type_tag_field + + # TODO: this type hinting... doesn't help much as it could return + # another cls + def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: + return lambda cls: self.register(cls=cls, tag=tag, field=field) + + def from_object(self, value): + return self.the_dict[type(value)] + + def from_tag(self, tag): + return self.the_dict[tag] + + +class AdjacentlyTaggedUnion(marshmallow.fields.Field): + def __init__( + self, + *, + from_object: typing.Callable[[typing.Any], TypeTagField], + from_tag: typing.Callable[[str], TypeTagField], + **kwargs, + ): + super().__init__(**kwargs) + + self.from_object = from_object + self.from_tag = from_tag + + def _deserialize( + self, + value: typing.Any, + attr: typing.Optional[str], + data: typing.Optional[typing.Mapping[str, typing.Any]], + **kwargs + ) -> typing.Any: + tag = value['type'] + serialized_value = value['value'] + type_tag_field = self.from_tag(tag) + field = type_tag_field.field() + + return field.deserialize(serialized_value) + + def _serialize( + self, + value: typing.Any, + attr: str, + obj: typing.Any, + **kwargs, + ) -> typing.Any: + type_tag_field = self.from_object(value) + field = type_tag_field.field() + tag = type_tag_field.tag + serialized_value = field.serialize(attr, obj) + + return {'type': tag, 'value': serialized_value} diff --git a/tests/test_fields.py b/tests/test_fields.py new file mode 100644 index 0000000..627428a --- /dev/null +++ b/tests/test_fields.py @@ -0,0 +1,74 @@ +import typing + +import attr +import marshmallow +import pytest + +import desert._fields + + +# TODO: test that field constructor doesn't tromple Field parameters + + +@attr.s(auto_attribs=True) +class ExampleData: + object: typing.Any + tag: str + field: typing.Type[marshmallow.fields.Field] + + +example_data = [ + ExampleData( + object=3.7, + tag='float_tag', + field=marshmallow.fields.Float, + ), +] + + +@pytest.fixture( + name='example_data', + params=example_data, +) +def _example_data(request): + return request.param + + +@pytest.fixture(name='registry', scope='session') +def _registry(): + registry = desert._fields.TypeDictRegistry() + + for example in example_data: + registry.register( + cls=type(example.object), + tag=example.tag, + field=example.field, + ) + + return registry + + +def test_adjacently_tagged_deserialize(example_data, registry): + field = desert._fields.AdjacentlyTaggedUnion( + from_object=registry.from_object, + from_tag=registry.from_tag, + ) + + serialized_value = {'type': example_data.tag, 'value': example_data.object} + + deserialized_value = field.deserialize(serialized_value) + + assert deserialized_value == example_data.object + + +def test_adjacently_tagged_serialize(example_data, registry): + field = desert._fields.AdjacentlyTaggedUnion( + from_object=registry.from_object, + from_tag=registry.from_tag, + ) + + obj = {'key': example_data.object} + + serialized_value = field.serialize('key', obj) + + assert serialized_value == {'type': example_data.tag, 'value': example_data.object} From 4f9834651feace3d056009967b955f4bde91ee21 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 12 Mar 2020 08:13:15 -0400 Subject: [PATCH 02/53] black --- src/desert/_fields.py | 41 ++++++++++++++++++----------------------- tests/test_fields.py | 29 ++++++++++------------------- 2 files changed, 28 insertions(+), 42 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 1272d37..7355ed0 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -4,7 +4,7 @@ import marshmallow.fields -T = typing.TypeVar('T') +T = typing.TypeVar("T") @attr.s(frozen=True, auto_attribs=True) @@ -16,10 +16,9 @@ class TypeTagField: @attr.s(auto_attribs=True) class TypeDictRegistry: - the_dict: typing.Dict[ - typing.Union[type, str], - marshmallow.fields.Field, - ] = attr.ib(factory=dict) + the_dict: typing.Dict[typing.Union[type, str], marshmallow.fields.Field,] = attr.ib( + factory=dict + ) def register(self, cls, tag, field): if any(key in self.the_dict for key in [cls, tag]): @@ -44,11 +43,11 @@ def from_tag(self, tag): class AdjacentlyTaggedUnion(marshmallow.fields.Field): def __init__( - self, - *, - from_object: typing.Callable[[typing.Any], TypeTagField], - from_tag: typing.Callable[[str], TypeTagField], - **kwargs, + self, + *, + from_object: typing.Callable[[typing.Any], TypeTagField], + from_tag: typing.Callable[[str], TypeTagField], + **kwargs, ): super().__init__(**kwargs) @@ -56,29 +55,25 @@ def __init__( self.from_tag = from_tag def _deserialize( - self, - value: typing.Any, - attr: typing.Optional[str], - data: typing.Optional[typing.Mapping[str, typing.Any]], - **kwargs + self, + value: typing.Any, + attr: typing.Optional[str], + data: typing.Optional[typing.Mapping[str, typing.Any]], + **kwargs, ) -> typing.Any: - tag = value['type'] - serialized_value = value['value'] + tag = value["type"] + serialized_value = value["value"] type_tag_field = self.from_tag(tag) field = type_tag_field.field() return field.deserialize(serialized_value) def _serialize( - self, - value: typing.Any, - attr: str, - obj: typing.Any, - **kwargs, + self, value: typing.Any, attr: str, obj: typing.Any, **kwargs, ) -> typing.Any: type_tag_field = self.from_object(value) field = type_tag_field.field() tag = type_tag_field.tag serialized_value = field.serialize(attr, obj) - return {'type': tag, 'value': serialized_value} + return {"type": tag, "value": serialized_value} diff --git a/tests/test_fields.py b/tests/test_fields.py index 627428a..bae9f21 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -18,31 +18,24 @@ class ExampleData: example_data = [ - ExampleData( - object=3.7, - tag='float_tag', - field=marshmallow.fields.Float, - ), + ExampleData(object=3.7, tag="float_tag", field=marshmallow.fields.Float,), ] @pytest.fixture( - name='example_data', - params=example_data, + name="example_data", params=example_data, ) def _example_data(request): return request.param -@pytest.fixture(name='registry', scope='session') +@pytest.fixture(name="registry", scope="session") def _registry(): registry = desert._fields.TypeDictRegistry() for example in example_data: registry.register( - cls=type(example.object), - tag=example.tag, - field=example.field, + cls=type(example.object), tag=example.tag, field=example.field, ) return registry @@ -50,11 +43,10 @@ def _registry(): def test_adjacently_tagged_deserialize(example_data, registry): field = desert._fields.AdjacentlyTaggedUnion( - from_object=registry.from_object, - from_tag=registry.from_tag, + from_object=registry.from_object, from_tag=registry.from_tag, ) - serialized_value = {'type': example_data.tag, 'value': example_data.object} + serialized_value = {"type": example_data.tag, "value": example_data.object} deserialized_value = field.deserialize(serialized_value) @@ -63,12 +55,11 @@ def test_adjacently_tagged_deserialize(example_data, registry): def test_adjacently_tagged_serialize(example_data, registry): field = desert._fields.AdjacentlyTaggedUnion( - from_object=registry.from_object, - from_tag=registry.from_tag, + from_object=registry.from_object, from_tag=registry.from_tag, ) - obj = {'key': example_data.object} + obj = {"key": example_data.object} - serialized_value = field.serialize('key', obj) + serialized_value = field.serialize("key", obj) - assert serialized_value == {'type': example_data.tag, 'value': example_data.object} + assert serialized_value == {"type": example_data.tag, "value": example_data.object} From 14c727301f88d3a39a8bab3e4c33e3ede3b61276 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 12 Mar 2020 08:33:34 -0400 Subject: [PATCH 03/53] raise on extra keys when deserializing adjacently tagged --- src/desert/_fields.py | 4 ++++ tests/test_fields.py | 34 +++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 7355ed0..30a1647 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -63,6 +63,10 @@ def _deserialize( ) -> typing.Any: tag = value["type"] serialized_value = value["value"] + + if len(value) > 2: + raise Exception() + type_tag_field = self.from_tag(tag) field = type_tag_field.field() diff --git a/tests/test_fields.py b/tests/test_fields.py index bae9f21..b339547 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -17,13 +17,13 @@ class ExampleData: field: typing.Type[marshmallow.fields.Field] -example_data = [ +example_data_list = [ ExampleData(object=3.7, tag="float_tag", field=marshmallow.fields.Float,), ] @pytest.fixture( - name="example_data", params=example_data, + name="example_data", params=example_data_list, ) def _example_data(request): return request.param @@ -33,7 +33,7 @@ def _example_data(request): def _registry(): registry = desert._fields.TypeDictRegistry() - for example in example_data: + for example in example_data_list: registry.register( cls=type(example.object), tag=example.tag, field=example.field, ) @@ -41,25 +41,37 @@ def _registry(): return registry -def test_adjacently_tagged_deserialize(example_data, registry): - field = desert._fields.AdjacentlyTaggedUnion( +@pytest.fixture(name="adjacently_tagged_field", scope="session") +def _adjacently_tagged_field(registry): + return desert._fields.AdjacentlyTaggedUnion( from_object=registry.from_object, from_tag=registry.from_tag, ) + +def test_adjacently_tagged_deserialize(example_data, adjacently_tagged_field): serialized_value = {"type": example_data.tag, "value": example_data.object} - deserialized_value = field.deserialize(serialized_value) + deserialized_value = adjacently_tagged_field.deserialize(serialized_value) assert deserialized_value == example_data.object -def test_adjacently_tagged_serialize(example_data, registry): - field = desert._fields.AdjacentlyTaggedUnion( - from_object=registry.from_object, from_tag=registry.from_tag, - ) +def test_adjacently_tagged_deserialize_extra_key_raises( + example_data, adjacently_tagged_field, +): + serialized_value = { + "type": example_data.tag, + "value": example_data.object, + "extra": 29, + } + + with pytest.raises(expected_exception=Exception): + adjacently_tagged_field.deserialize(serialized_value) + +def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): obj = {"key": example_data.object} - serialized_value = field.serialize("key", obj) + serialized_value = adjacently_tagged_field.serialize("key", obj) assert serialized_value == {"type": example_data.tag, "value": example_data.object} From cd0f7b6ec701d7d008f9a326dcea55f479efcc05 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 12 Mar 2020 08:46:32 -0400 Subject: [PATCH 04/53] remove trailing comma on single line, thanks black --- src/desert/_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 30a1647..17da2a5 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -16,7 +16,7 @@ class TypeTagField: @attr.s(auto_attribs=True) class TypeDictRegistry: - the_dict: typing.Dict[typing.Union[type, str], marshmallow.fields.Field,] = attr.ib( + the_dict: typing.Dict[typing.Union[type, str], marshmallow.fields.Field] = attr.ib( factory=dict ) From 08c6e0cc33ba4e0119327b68baa36ac0382e7b5f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 12 Mar 2020 08:47:58 -0400 Subject: [PATCH 05/53] correct TypeTagField.field type hint --- src/desert/_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 17da2a5..533231b 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -11,7 +11,7 @@ class TypeTagField: cls: type tag: str - field: marshmallow.fields.Field + field: typing.Type[marshmallow.fields.Field] @attr.s(auto_attribs=True) From ddb5df40b7328c61c3a41d572ff10615f54eca5f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 12 Mar 2020 13:20:48 -0400 Subject: [PATCH 06/53] add str and decimal.Decimal examples --- tests/test_fields.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index b339547..bdc5174 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,3 +1,4 @@ +import decimal import typing import attr @@ -14,16 +15,24 @@ class ExampleData: object: typing.Any tag: str - field: typing.Type[marshmallow.fields.Field] + field: typing.Callable[[], marshmallow.fields.Field] example_data_list = [ - ExampleData(object=3.7, tag="float_tag", field=marshmallow.fields.Float,), + ExampleData(object=3.7, tag="float_tag", field=marshmallow.fields.Float), + ExampleData(object="29", tag="str_tag", field=marshmallow.fields.String), + ExampleData( + object=decimal.Decimal("4.2"), + tag="decimal_tag", + field=marshmallow.fields.Decimal, + ), ] @pytest.fixture( - name="example_data", params=example_data_list, + name="example_data", + params=example_data_list, + ids=[str(example) for example in example_data_list], ) def _example_data(request): return request.param @@ -53,7 +62,9 @@ def test_adjacently_tagged_deserialize(example_data, adjacently_tagged_field): deserialized_value = adjacently_tagged_field.deserialize(serialized_value) - assert deserialized_value == example_data.object + assert (type(deserialized_value) == type(example_data.object)) and ( + deserialized_value == example_data.object + ) def test_adjacently_tagged_deserialize_extra_key_raises( From 24925190267d8677df03455909ce3f1d55bad484 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 12 Mar 2020 13:22:21 -0400 Subject: [PATCH 07/53] break by adding a couple of different list examples --- tests/test_fields.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_fields.py b/tests/test_fields.py index bdc5174..98077c6 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -26,6 +26,16 @@ class ExampleData: tag="decimal_tag", field=marshmallow.fields.Decimal, ), + ExampleData( + object=[1, 2, 3], + tag="integer_list_tag", + field=lambda: marshmallow.fields.List(marshmallow.fields.Integer()), + ), + ExampleData( + object=['abc', '2', 'mno'], + tag="string_list_tag", + field=lambda: marshmallow.fields.List(marshmallow.fields.String()), + ), ] From 1ce7e67e20290cf989dbdd0cd2c6e319db03fa5d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 12 Mar 2020 20:57:56 -0400 Subject: [PATCH 08/53] remove explicit registry collision check this gives much more interesting test results where you can see the deserialization is fine due to the unique tags but the serialization is not. --- src/desert/_fields.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 533231b..f1731f3 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -21,8 +21,9 @@ class TypeDictRegistry: ) def register(self, cls, tag, field): - if any(key in self.the_dict for key in [cls, tag]): - raise Exception() + # TODO: just disabling for now to show more interesting test results + # if any(key in self.the_dict for key in [cls, tag]): + # raise Exception() type_tag_field = TypeTagField(cls=cls, tag=tag, field=field) From cac94deb72e300e921b48c33e99bd6a07d77bbaa Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 12 Mar 2020 23:06:52 -0400 Subject: [PATCH 09/53] correct tests, better demonstrate failures, prep for additional registry types --- src/desert/_fields.py | 6 +-- tests/test_fields.py | 113 +++++++++++++++++++++++++++++++----------- 2 files changed, 87 insertions(+), 32 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index f1731f3..e17b28d 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -15,7 +15,7 @@ class TypeTagField: @attr.s(auto_attribs=True) -class TypeDictRegistry: +class TypeDictFieldRegistry: the_dict: typing.Dict[typing.Union[type, str], marshmallow.fields.Field] = attr.ib( factory=dict ) @@ -69,7 +69,7 @@ def _deserialize( raise Exception() type_tag_field = self.from_tag(tag) - field = type_tag_field.field() + field = type_tag_field.field return field.deserialize(serialized_value) @@ -77,7 +77,7 @@ def _serialize( self, value: typing.Any, attr: str, obj: typing.Any, **kwargs, ) -> typing.Any: type_tag_field = self.from_object(value) - field = type_tag_field.field() + field = type_tag_field.field tag = type_tag_field.tag serialized_value = field.serialize(attr, obj) diff --git a/tests/test_fields.py b/tests/test_fields.py index 98077c6..0615856 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -10,31 +10,65 @@ # TODO: test that field constructor doesn't tromple Field parameters +_NOTHING = object() + @attr.s(auto_attribs=True) class ExampleData: - object: typing.Any + to_serialize: typing.Any + serialized: typing.Any + deserialized: typing.Any tag: str - field: typing.Callable[[], marshmallow.fields.Field] + field: marshmallow.fields.Field + + @classmethod + def build( + cls, to_serialize, tag, field, serialized=_NOTHING, deserialized=_NOTHING, + ): + if serialized is _NOTHING: + serialized = to_serialize + + if deserialized is _NOTHING: + deserialized = to_serialize + + return cls( + to_serialize=to_serialize, + serialized=serialized, + deserialized=deserialized, + tag=tag, + field=field, + ) example_data_list = [ - ExampleData(object=3.7, tag="float_tag", field=marshmallow.fields.Float), - ExampleData(object="29", tag="str_tag", field=marshmallow.fields.String), - ExampleData( - object=decimal.Decimal("4.2"), + ExampleData.build( + to_serialize=3.7, tag="float_tag", field=marshmallow.fields.Float() + ), + ExampleData.build( + to_serialize="29", tag="str_tag", field=marshmallow.fields.String() + ), + ExampleData.build( + to_serialize=decimal.Decimal("4.2"), + serialized="4.2", tag="decimal_tag", - field=marshmallow.fields.Decimal, + field=marshmallow.fields.Decimal(as_string=True), ), - ExampleData( - object=[1, 2, 3], + ExampleData.build( + to_serialize=[1, 2, 3], tag="integer_list_tag", - field=lambda: marshmallow.fields.List(marshmallow.fields.Integer()), + field=marshmallow.fields.List(marshmallow.fields.Integer()), + ), + ExampleData.build( + to_serialize=["abc", "2", "mno"], + tag="string_list_tag", + field=marshmallow.fields.List(marshmallow.fields.String()), ), ExampleData( - object=['abc', '2', 'mno'], + to_serialize=("def", "13"), + serialized=["def", "13"], + deserialized=["def", "13"], tag="string_list_tag", - field=lambda: marshmallow.fields.List(marshmallow.fields.String()), + field=marshmallow.fields.List(marshmallow.fields.String()), ), ] @@ -48,19 +82,40 @@ def _example_data(request): return request.param -@pytest.fixture(name="registry", scope="session") -def _registry(): - registry = desert._fields.TypeDictRegistry() +def build_type_dict_registry(examples): + registry = desert._fields.TypeDictFieldRegistry() - for example in example_data_list: + for example in examples: registry.register( - cls=type(example.object), tag=example.tag, field=example.field, + cls=type(example.deserialized), tag=example.tag, field=example.field, ) return registry -@pytest.fixture(name="adjacently_tagged_field", scope="session") +registry_builders = [ + build_type_dict_registry, +] +registries = [ + registry_builder(example_data_list) + for registry_builder in registry_builders +] +registry_ids = [ + type(registry).__name__ + for registry in registries +] + + +@pytest.fixture( + name="registry", + params=registries, + ids=registry_ids, +) +def _registry(request): + return request.param + + +@pytest.fixture(name="adjacently_tagged_field") def _adjacently_tagged_field(registry): return desert._fields.AdjacentlyTaggedUnion( from_object=registry.from_object, from_tag=registry.from_tag, @@ -68,31 +123,31 @@ def _adjacently_tagged_field(registry): def test_adjacently_tagged_deserialize(example_data, adjacently_tagged_field): - serialized_value = {"type": example_data.tag, "value": example_data.object} + serialized = {"type": example_data.tag, "value": example_data.serialized} - deserialized_value = adjacently_tagged_field.deserialize(serialized_value) + deserialized = adjacently_tagged_field.deserialize(serialized) - assert (type(deserialized_value) == type(example_data.object)) and ( - deserialized_value == example_data.object - ) + expected = example_data.deserialized + + assert (type(deserialized) == type(expected)) and (deserialized == expected) def test_adjacently_tagged_deserialize_extra_key_raises( example_data, adjacently_tagged_field, ): - serialized_value = { + serialized = { "type": example_data.tag, - "value": example_data.object, + "value": example_data.serialized, "extra": 29, } with pytest.raises(expected_exception=Exception): - adjacently_tagged_field.deserialize(serialized_value) + adjacently_tagged_field.deserialize(serialized) def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): - obj = {"key": example_data.object} + obj = {"key": example_data.to_serialize} - serialized_value = adjacently_tagged_field.serialize("key", obj) + serialized = adjacently_tagged_field.serialize("key", obj) - assert serialized_value == {"type": example_data.tag, "value": example_data.object} + assert serialized == {"type": example_data.tag, "value": example_data.serialized} From f2cffbeebe667b15c6455fef2ab61b4ab83cfb39 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 12 Mar 2020 23:19:29 -0400 Subject: [PATCH 10/53] black... --- tests/test_fields.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index 0615856..9d10e8d 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -97,19 +97,13 @@ def build_type_dict_registry(examples): build_type_dict_registry, ] registries = [ - registry_builder(example_data_list) - for registry_builder in registry_builders -] -registry_ids = [ - type(registry).__name__ - for registry in registries + registry_builder(example_data_list) for registry_builder in registry_builders ] +registry_ids = [type(registry).__name__ for registry in registries] @pytest.fixture( - name="registry", - params=registries, - ids=registry_ids, + name="registry", params=registries, ids=registry_ids, ) def _registry(request): return request.param From 84cd5ed2cde80b13801ddec84ab8f42b1393b23b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 29 May 2020 23:34:12 -0400 Subject: [PATCH 11/53] maybe interesting --- src/desert/_fields.py | 121 ++++++++++++++++++++++++++++++++++++------ tests/test_fields.py | 81 +++++++++++++++++++++++++--- 2 files changed, 179 insertions(+), 23 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index e17b28d..f6316ec 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -2,52 +2,143 @@ import attr import marshmallow.fields +import pytypes T = typing.TypeVar("T") @attr.s(frozen=True, auto_attribs=True) -class TypeTagField: - cls: type +class HintTagField: + hint: typing.Any tag: str - field: typing.Type[marshmallow.fields.Field] + field: marshmallow.fields.Field + + +class FieldRegistry(typing.Protocol): + def from_object(self, value: typing.Any) -> marshmallow.fields.Field: + ... + + def from_tag(self, tag: str) -> marshmallow.fields.Field: + ... @attr.s(auto_attribs=True) class TypeDictFieldRegistry: - the_dict: typing.Dict[typing.Union[type, str], marshmallow.fields.Field] = attr.ib( + the_dict: typing.Dict[ + typing.Union[type, str], + marshmallow.fields.Field, + ] = attr.ib( factory=dict ) - def register(self, cls, tag, field): + def register( + self, + hint: typing.Any, + tag: str, + field: marshmallow.fields.Field, + ) -> None: # TODO: just disabling for now to show more interesting test results # if any(key in self.the_dict for key in [cls, tag]): # raise Exception() - type_tag_field = TypeTagField(cls=cls, tag=tag, field=field) + type_tag_field = HintTagField(hint=hint, tag=tag, field=field) - self.the_dict[cls] = type_tag_field + self.the_dict[hint] = type_tag_field self.the_dict[tag] = type_tag_field - # TODO: this type hinting... doesn't help much as it could return - # another cls - def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: - return lambda cls: self.register(cls=cls, tag=tag, field=field) + # # TODO: this type hinting... doesn't help much as it could return + # # another cls + # def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: + # return lambda cls: self.register(cls=cls, tag=tag, field=field) - def from_object(self, value): + def from_object(self, value: typing.Any) -> marshmallow.fields.Field: return self.the_dict[type(value)] - def from_tag(self, tag): + def from_tag(self, tag: str) -> marshmallow.fields.Field: return self.the_dict[tag] +@attr.s(auto_attribs=True) +class OrderedIsinstanceFieldRegistry: + the_list: typing.List[HintTagField] = attr.ib(factory=list) + by_tag: typing.Dict[str, HintTagField] = attr.ib(factory=dict) + + # TODO: but type bans from-scratch metatypes... and protocols + def register( + self, + hint: typing.Any, + tag: str, + field: marshmallow.fields.Field, + ) -> None: + # TODO: just disabling for now to show more interesting test results + # if any(key in self.the_dict for key in [cls, tag]): + # raise Exception() + + type_tag_field = HintTagField(hint=hint, tag=tag, field=field) + + self.the_list.append(type_tag_field) + self.by_tag[tag] = type_tag_field + + # # TODO: this type hinting... doesn't help much as it could return + # # another cls + # def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: + # return lambda cls: self.register(cls=cls, tag=tag, field=field) + + def from_object(self, value: typing.Any) -> HintTagField: + for type_tag_field in self.the_list: + if isinstance(value, type_tag_field.cls): + return type_tag_field + + raise Exception() + + def from_tag(self, tag: str) -> HintTagField: + return self.by_tag[tag] + + +@attr.s(auto_attribs=True) +class OrderedIsinstanceFieldRegistry: + the_list: typing.List[HintTagField] = attr.ib(factory=list) + by_tag: typing.Dict[str, HintTagField] = attr.ib(factory=dict) + + # TODO: but type bans from-scratch metatypes... and protocols + def register( + self, + hint: typing.Any, + tag: str, + field: marshmallow.fields.Field, + ) -> None: + # TODO: just disabling for now to show more interesting test results + # if any(key in self.the_dict for key in [cls, tag]): + # raise Exception() + + type_tag_field = HintTagField(hint=hint, tag=tag, field=field) + + self.the_list.append(type_tag_field) + self.by_tag[tag] = type_tag_field + + # # TODO: this type hinting... doesn't help much as it could return + # # another cls + # def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: + # return lambda cls: self.register(cls=cls, tag=tag, field=field) + + def from_object(self, value: typing.Any) -> HintTagField: + for type_tag_field in self.the_list: + if pytypes.is_of_type(value, type_tag_field.hint): + return type_tag_field + + raise Exception() + + def from_tag(self, tag: str) -> HintTagField: + return self.by_tag[tag] + + class AdjacentlyTaggedUnion(marshmallow.fields.Field): def __init__( self, *, - from_object: typing.Callable[[typing.Any], TypeTagField], - from_tag: typing.Callable[[str], TypeTagField], + from_object: typing.Callable[[typing.Any], HintTagField], + from_tag: typing.Callable[[str], HintTagField], **kwargs, ): super().__init__(**kwargs) diff --git a/tests/test_fields.py b/tests/test_fields.py index 9d10e8d..a045a06 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,3 +1,5 @@ +import abc +import collections.abc import decimal import typing @@ -20,10 +22,12 @@ class ExampleData: deserialized: typing.Any tag: str field: marshmallow.fields.Field + # TODO: can we be more specific? + hint: typing.Any @classmethod def build( - cls, to_serialize, tag, field, serialized=_NOTHING, deserialized=_NOTHING, + cls, hint, to_serialize, tag, field, serialized=_NOTHING, deserialized=_NOTHING, ): if serialized is _NOTHING: serialized = to_serialize @@ -32,6 +36,7 @@ def build( deserialized = to_serialize return cls( + hint=hint, to_serialize=to_serialize, serialized=serialized, deserialized=deserialized, @@ -42,28 +47,38 @@ def build( example_data_list = [ ExampleData.build( - to_serialize=3.7, tag="float_tag", field=marshmallow.fields.Float() + hint=float, + to_serialize=3.7, + tag="float_tag", + field=marshmallow.fields.Float(), ), ExampleData.build( - to_serialize="29", tag="str_tag", field=marshmallow.fields.String() + hint=str, + to_serialize="29", + tag="str_tag", + field=marshmallow.fields.String(), ), ExampleData.build( + hint=decimal.Decimal, to_serialize=decimal.Decimal("4.2"), serialized="4.2", tag="decimal_tag", field=marshmallow.fields.Decimal(as_string=True), ), ExampleData.build( + hint=typing.List[int], to_serialize=[1, 2, 3], tag="integer_list_tag", field=marshmallow.fields.List(marshmallow.fields.Integer()), ), ExampleData.build( + hint=typing.List[str], to_serialize=["abc", "2", "mno"], tag="string_list_tag", field=marshmallow.fields.List(marshmallow.fields.String()), ), ExampleData( + hint=typing.Sequence[str], to_serialize=("def", "13"), serialized=["def", "13"], deserialized=["def", "13"], @@ -87,17 +102,67 @@ def build_type_dict_registry(examples): for example in examples: registry.register( - cls=type(example.deserialized), tag=example.tag, field=example.field, + hint=example.hint, tag=example.tag, field=example.field, + ) + + return registry + + +# class NonStringSequence(abc.ABC): +# @classmethod +# def __subclasshook__(cls, maybe_subclass): +# return isinstance(maybe_subclass, collections.abc.Sequence) and not isinstance( +# maybe_subclass, str +# ) + + +def build_order_isinstance_registry(examples): + registry = desert._fields.OrderedIsinstanceFieldRegistry() + + # registry.register( + # hint=typing.List[], + # tag="sequence_abc", + # field=marshmallow.fields.List(marshmallow.fields.String()), + # ) + + for example in examples: + registry.register( + hint=example.hint, + tag=example.tag, + field=example.field, + ) + + return registry + + +# class NonStringSequence(abc.ABC): +# @classmethod +# def __subclasshook__(cls, maybe_subclass): +# return isinstance(maybe_subclass, collections.abc.Sequence) and not isinstance( +# maybe_subclass, str +# ) + + +def build_order_isinstance_registry(examples): + registry = desert._fields.OrderedIsinstanceFieldRegistry() + + # registry.register( + # hint=typing.Sequence, + # tag="sequence_abc", + # field=marshmallow.fields.List(marshmallow.fields.String()), + # ) + + for example in examples: + registry.register( + hint=example.hint, tag=example.tag, field=example.field, ) return registry -registry_builders = [ - build_type_dict_registry, -] registries = [ - registry_builder(example_data_list) for registry_builder in registry_builders + # build_type_dict_registry(example_data_list), + build_order_isinstance_registry(example_data_list), ] registry_ids = [type(registry).__name__ for registry in registries] From e76a271db7a6571ac36cafc4fe66fa7f2b83e3ad Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 22:35:32 -0400 Subject: [PATCH 12/53] use typeguard instead, for now at least --- dev-requirements.txt | 203 +++++++++++++++++++++++++++++++++++++----- requirements.in | 2 + requirements.txt | 16 +++- src/desert/_fields.py | 16 +++- test-requirements.txt | 46 +++++++--- 5 files changed, 238 insertions(+), 45 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index cde7294..ce189f5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,10 +4,13 @@ # # python pycli lock # +ansicolors==1.0.2 \ + --hash=sha256:7664530bb992e3847b61e3aab1580b4df9ed00c5898e80194a9933bc9c80950a \ + # via cuvner appdirs==1.4.3 \ --hash=sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92 \ --hash=sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e \ - # via black + # via black, virtualenv astroid==2.2.5 \ --hash=sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4 \ --hash=sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4 \ @@ -15,16 +18,18 @@ astroid==2.2.5 \ attrs==19.1.0 \ --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79 \ --hash=sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399 \ - # via black + # via -r requirements.in, black, pytest black==19.10b0 \ --hash=sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b \ - --hash=sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539 + --hash=sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539 \ + # via -r dev-requirements.in bleach==3.1.1 \ --hash=sha256:44f69771e2ac81ff30d929d485b7f9919f3ad6d019b6b20c74f3b8687c3f70df \ --hash=sha256:aa8b870d0f46965bac2c073a93444636b0e1ca74e9777e34f03dd494b8a59d48 \ # via readme-renderer bump2version==0.5.10 \ - --hash=sha256:185abfd0d8321ec5059424d8b670aa82f7385948ff7ddd986981b4ed04dc819a + --hash=sha256:185abfd0d8321ec5059424d8b670aa82f7385948ff7ddd986981b4ed04dc819a \ + # via -r dev-requirements.in certifi==2019.11.28 \ --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \ --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f \ @@ -70,11 +75,45 @@ chardet==3.0.4 \ # via requests check-manifest==0.39 \ --hash=sha256:8754cc8efd7c062a3705b442d1c23ff702d4477b41a269c2e354b25e1f5535a4 \ - --hash=sha256:a4c555f658a7c135b8a22bd26c2e55cfaf5876e4d5962d8c25652f2addd556bc + --hash=sha256:a4c555f658a7c135b8a22bd26c2e55cfaf5876e4d5962d8c25652f2addd556bc \ + # via -r dev-requirements.in click==7.0 \ --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ - # via black, towncrier + # via black, cuvner, towncrier +coverage==5.1 \ + --hash=sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a \ + --hash=sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355 \ + --hash=sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65 \ + --hash=sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7 \ + --hash=sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9 \ + --hash=sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1 \ + --hash=sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0 \ + --hash=sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55 \ + --hash=sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c \ + --hash=sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6 \ + --hash=sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef \ + --hash=sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019 \ + --hash=sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e \ + --hash=sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0 \ + --hash=sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf \ + --hash=sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24 \ + --hash=sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2 \ + --hash=sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c \ + --hash=sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4 \ + --hash=sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0 \ + --hash=sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd \ + --hash=sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04 \ + --hash=sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e \ + --hash=sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730 \ + --hash=sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2 \ + --hash=sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768 \ + --hash=sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796 \ + --hash=sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7 \ + --hash=sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a \ + --hash=sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489 \ + --hash=sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052 \ + # via -r test-requirements.in, cuvner, pytest-cov cryptography==2.8 \ --hash=sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c \ --hash=sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595 \ @@ -98,21 +137,41 @@ cryptography==2.8 \ --hash=sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf \ --hash=sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8 \ # via secretstorage +cuvner==18.0.1 \ + --hash=sha256:395b9b1d802aca999212e70566813b79c6a50289d6a2a712252c9c9eb52cf29e \ + # via -r test-requirements.in +dataclasses==0.6 \ + --hash=sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f \ + --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84 \ + # via -r requirements.in +distlib==0.3.0 \ + --hash=sha256:2e166e231a26b36d6dfe35a48c4464346620f8645ed0ace01ee31822b288de21 \ + # via virtualenv docutils==0.14 \ --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \ --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \ - --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 + --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \ + # via -r dev-requirements.in, readme-renderer +filelock==3.0.12 \ + --hash=sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59 \ + --hash=sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836 \ + # via tox, virtualenv idna==2.8 \ --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ # via requests +importlib-metadata==1.6.0 \ + --hash=sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f \ + --hash=sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e \ + # via -r test-requirements.in incremental==17.5.0 \ --hash=sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f \ --hash=sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3 \ - # via towncrier + # via cuvner, towncrier isort==4.3.20 \ --hash=sha256:c40744b6bc5162bbb39c1257fe298b7a393861d50978b565f3ccd9cb9de0182a \ - --hash=sha256:f57abacd059dc3bd666258d1efb0377510a89777fda3e3274e3c01f7c03ae22d + --hash=sha256:f57abacd059dc3bd666258d1efb0377510a89777fda3e3274e3c01f7c03ae22d \ + # via -r dev-requirements.in, pylint jeepney==0.4.2 \ --hash=sha256:0ba6d8c597e9bef1ebd18aaec595f942a264e25c1a48f164d46120eacaa2e9bb \ --hash=sha256:6f45dce1125cf6c58a1c88123d3831f36a789f9204fbad3172eac15f8ccd08d0 \ @@ -175,14 +234,30 @@ markupsafe==1.1.1 \ --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \ --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \ # via jinja2 +marshmallow-enum==1.5.1 \ + --hash=sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58 \ + --hash=sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072 \ + # via -r test-requirements.in +marshmallow-union==0.1.15 \ + --hash=sha256:711d4e3e7085ab9110bd29b63d9fd1b6a2d42a752909892ae6463865a5b2b226 \ + --hash=sha256:c4a03373e4d60f338a4468df496af5829276e4c13d5b900501528caf392089d9 \ + # via -r test-requirements.in +marshmallow==3.6.0 \ + --hash=sha256:c2673233aa21dde264b84349dc2fd1dce5f30ed724a0a00e75426734de5b84ab \ + --hash=sha256:f88fe96434b1f0f476d54224d59333eba8ca1a203a2695683c1855675c4049a7 \ + # via -r requirements.in, marshmallow-enum, marshmallow-union mccabe==0.6.1 \ --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \ # via pylint +more-itertools==8.3.0 \ + --hash=sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be \ + --hash=sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982 \ + # via pytest mypy-extensions==0.4.1 \ --hash=sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812 \ --hash=sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e \ - # via mypy + # via mypy, typing-inspect mypy==0.710 \ --hash=sha256:0c00f90cb20d3b5a7fa9b190e92e8a94cd4bbb26dd58bbed13bfc76677a39700 \ --hash=sha256:104086c924b1ac605bed1539886b86202c02b08f78649b45881b4a624ff66a46 \ @@ -194,30 +269,69 @@ mypy==0.710 \ --hash=sha256:67a632ea4596b417ed9af60fde828f6c6bb7e83e7f7f0d7057a3a1dece940199 \ --hash=sha256:953e5e10203df8691fcd8ce40a5e6e2ec37144b9fb5adf6bcbafc6b74bda0593 \ --hash=sha256:9e80c47b3d621cf8f98e58cc202a03a92b430a4d736129197a2a83cbe35a3243 \ - --hash=sha256:d0e3c21620637b1548c1e0f0c2b56617dd0a9dc1fda1afa00210db5922487cd3 + --hash=sha256:d0e3c21620637b1548c1e0f0c2b56617dd0a9dc1fda1afa00210db5922487cd3 \ + # via -r dev-requirements.in +packaging==20.4 \ + --hash=sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8 \ + --hash=sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181 \ + # via pytest, tox pathspec==0.7.0 \ --hash=sha256:163b0632d4e31cef212976cf57b43d9fd6b0bac6e67c26015d611a647d5e7424 \ --hash=sha256:562aa70af2e0d434367d9790ad37aed893de47f1693e4201fd1d3dca15d19b96 \ # via black +pathtools==0.1.2 \ + --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 \ + # via watchdog pex==1.6.7 \ --hash=sha256:6ada49d45b6a141d2e27a0ea60aa7f46432a8cde9db06049792a3d64af7dfb69 \ - --hash=sha256:dc95eed8047a6644a21b9814841efb31447aaeae587c3835c9262db29007e3c1 + --hash=sha256:dc95eed8047a6644a21b9814841efb31447aaeae587c3835c9262db29007e3c1 \ + # via -r dev-requirements.in pkginfo==1.5.0.1 \ --hash=sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb \ --hash=sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32 \ # via twine +pluggy==0.13.1 \ + --hash=sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0 \ + --hash=sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d \ + # via pytest, tox +py==1.8.1 \ + --hash=sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa \ + --hash=sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0 \ + # via pytest, tox pycparser==2.19 \ --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ # via cffi pygments==2.4.2 \ --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \ - --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 + --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 \ + # via -r dev-requirements.in, cuvner, readme-renderer pylint==2.3.1 \ --hash=sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09 \ - --hash=sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1 + --hash=sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1 \ + # via -r dev-requirements.in +pyparsing==2.4.7 \ + --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ + --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \ + # via packaging +pytest-cov==2.9.0 \ + --hash=sha256:b6a814b8ed6247bd81ff47f038511b57fe1ce7f4cc25b9106f1a4b106f1d9322 \ + --hash=sha256:c87dfd8465d865655a8213859f1b4749b43448b5fae465cb981e16d52a811424 \ + # via -r test-requirements.in +pytest-sphinx==0.2.2 \ + --hash=sha256:3f2f55af893f717c9befc5aa6dbcaeda3174258706dfd3dffb4cfe9a1671f8ed \ + # via -r test-requirements.in +pytest-travis-fold==1.3.0 \ + --hash=sha256:3fe15aa21ed14275e5a77814339281b3b618e350b98a43e7ac5d5bdcad8202cb \ + --hash=sha256:5607df571232b257be644400be559afb9148af3a27576f8080f56cee915771b2 \ + # via -r test-requirements.in +pytest==5.4.2 \ + --hash=sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3 \ + --hash=sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698 \ + # via -r test-requirements.in, pytest-cov, pytest-sphinx, pytest-travis-fold readme-renderer==24.0 \ --hash=sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f \ - --hash=sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d + --hash=sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d \ + # via -r dev-requirements.in, twine regex==2020.1.8 \ --hash=sha256:07b39bf943d3d2fe63d46281d8504f8df0ff3fe4c57e13d1656737950e53e525 \ --hash=sha256:0932941cdfb3afcbc26cc3bcf7c3f3d73d5a9b9c56955d432dbf8bbc147d4c5b \ @@ -253,24 +367,30 @@ secretstorage==3.1.2 \ --hash=sha256:15da8a989b65498e29be338b3b279965f1b8f09b9668bd8010da183024c8bff6 \ --hash=sha256:b5ec909dde94d4ae2fa26af7c089036997030f0cf0a5cb372b4cccabd81c143b \ # via keyring -six==1.12.0 \ - --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ - --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ - # via astroid, bleach, cryptography, readme-renderer +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \ + # via astroid, bleach, cryptography, cuvner, packaging, readme-renderer, tox, virtualenv toml==0.10.0 \ --hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \ --hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e \ - # via black, check-manifest, towncrier + # via black, check-manifest, towncrier, tox towncrier==19.2.0 \ --hash=sha256:48251a1ae66d2cf7e6fa5552016386831b3e12bb3b2d08eb70374508c17a8196 \ - --hash=sha256:de19da8b8cb44f18ea7ed3a3823087d2af8fcf497151bb9fd1e1b092ff56ed8d + --hash=sha256:de19da8b8cb44f18ea7ed3a3823087d2af8fcf497151bb9fd1e1b092ff56ed8d \ + # via -r dev-requirements.in +tox==3.15.1 \ + --hash=sha256:322dfdf007d7d53323f767badcb068a5cfa7c44d8aabb698d131b28cf44e62c4 \ + --hash=sha256:8c9ad9b48659d291c5bc78bcabaa4d680d627687154b812fa52baedaa94f9f83 \ + # via -r test-requirements.in tqdm==4.41.1 \ --hash=sha256:4789ccbb6fc122b5a6a85d512e4e41fc5acad77216533a6f2b8ce51e0f265c23 \ --hash=sha256:efab950cf7cc1e4d8ee50b2bb9c8e4a89f8307b49e0b2c9cfef3ec4ca26655eb \ # via twine twine==3.1.1 \ --hash=sha256:c1af8ca391e43b0a06bbc155f7f67db0bf0d19d284bfc88d1675da497a946124 \ - --hash=sha256:d561a5e511f70275e5a485a6275ff61851c16ffcb3a95a602189161112d9f160 + --hash=sha256:d561a5e511f70275e5a485a6275ff61851c16ffcb3a95a602189161112d9f160 \ + # via -r dev-requirements.in typed-ast==1.4.0 \ --hash=sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161 \ --hash=sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e \ @@ -293,23 +413,58 @@ typed-ast==1.4.0 \ --hash=sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66 \ --hash=sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12 \ # via astroid, black, mypy +typeguard==2.7.1 \ + --hash=sha256:1d3710251d3d3d6c64e0c49f45edec2e88ddc386a51e89c3ec0703efeb8b3b81 \ + --hash=sha256:2d545c71e9439c21bcd7c28f5f55b3606e6106f7031ab58375656a1aed483ef2 \ + # via -r requirements.in +typing-extensions==3.7.4.2 \ + --hash=sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5 \ + --hash=sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae \ + --hash=sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392 \ + # via typing-inspect +typing-inspect==0.6.0 \ + --hash=sha256:3b98390df4d999a28cf5b35d8b333425af5da2ece8a4ea9e98f71e7591347b4f \ + --hash=sha256:8f1b1dd25908dbfd81d3bebc218011531e7ab614ba6e5bf7826d887c834afab7 \ + --hash=sha256:de08f50a22955ddec353876df7b2545994d6df08a2f45d54ac8c05e530372ca0 \ + # via -r requirements.in +unidiff==0.6.0 \ + --hash=sha256:90c5214e9a357ff4b2fee19d91e77706638e3e00592a732d9405ea4e93da981f \ + --hash=sha256:e1dd956a492ccc4351e24931b2f2d29c79e3be17a99dd8f14e95324321d93a88 \ + # via cuvner urllib3==1.25.8 \ --hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \ --hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc \ # via requests versioneer==0.18 \ --hash=sha256:08e395c0acc544f78645b9c0ebfccaf47950ae61e0c85bd1aaea98ff59609aeb \ - --hash=sha256:ead1f78168150011189521b479d3a0dd2f55c94f5b07747b484fd693c3fbf335 + --hash=sha256:ead1f78168150011189521b479d3a0dd2f55c94f5b07747b484fd693c3fbf335 \ + # via -r dev-requirements.in +virtualenv==20.0.21 \ + --hash=sha256:a116629d4e7f4d03433b8afa27f43deba09d48bc48f5ecefa4f015a178efb6cf \ + --hash=sha256:a730548b27366c5e6cbdf6f97406d861cccece2e22275e8e1a757aeff5e00c70 \ + # via tox +watchdog==0.10.2 \ + --hash=sha256:c560efb643faed5ef28784b2245cf8874f939569717a4a12826a173ac644456b \ + # via cuvner +wcwidth==0.1.9 \ + --hash=sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1 \ + --hash=sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1 \ + # via pytest webencodings==0.5.1 \ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 \ # via bleach wheel==0.33.6 \ --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \ - --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28 + --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28 \ + # via -r dev-requirements.in wrapt==1.11.2 \ --hash=sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1 \ # via astroid +zipp==3.1.0 \ + --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \ + --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 \ + # via importlib-metadata # WARNING: The following packages were not pinned, but pip requires them to be # pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag. diff --git a/requirements.in b/requirements.in index ade396f..6b69309 100644 --- a/requirements.in +++ b/requirements.in @@ -2,3 +2,5 @@ marshmallow>=3.0 attrs typing_inspect dataclasses +#https://github.com/Stewori/pytypes/archive/b7271ec654d3553894febc6e0d8ad1b0e1ac570a.zip +typeguard diff --git a/requirements.txt b/requirements.txt index 79e3f60..0b53f45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,18 +6,26 @@ # attrs==19.1.0 \ --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79 \ - --hash=sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399 + --hash=sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399 \ + # via -r requirements.in dataclasses==0.6 \ --hash=sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f \ - --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84 + --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84 \ + # via -r requirements.in marshmallow==3.2.1 \ --hash=sha256:077b4612f5d3b9333b736fdc6b963d2b46d409070f44ff3e6c4109645c673e83 \ - --hash=sha256:9a2f3e8ea5f530a9664e882d7d04b58650f46190178b2264c72b7d20399d28f0 + --hash=sha256:9a2f3e8ea5f530a9664e882d7d04b58650f46190178b2264c72b7d20399d28f0 \ + # via -r requirements.in mypy-extensions==0.4.1 \ --hash=sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812 \ --hash=sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e \ # via typing-inspect +typeguard==2.7.1 \ + --hash=sha256:1d3710251d3d3d6c64e0c49f45edec2e88ddc386a51e89c3ec0703efeb8b3b81 \ + --hash=sha256:2d545c71e9439c21bcd7c28f5f55b3606e6106f7031ab58375656a1aed483ef2 \ + # via -r requirements.in typing-inspect==0.4.0 \ --hash=sha256:a7cb36c4a47d034766a67ea6467b39bd995cd00db8d4db1aa40001bf2d674a9b \ --hash=sha256:cf41eb276cc8955a45e03c15cd1efa6c181a8775a38ff0bfda99d28af97bcda3 \ - --hash=sha256:e319dfa0c9a646614c9b6abab3bdd5f860a98609998d420f33e41a6e01cbbddb + --hash=sha256:e319dfa0c9a646614c9b6abab3bdd5f860a98609998d420f33e41a6e01cbbddb \ + # via -r requirements.in diff --git a/src/desert/_fields.py b/src/desert/_fields.py index f6316ec..3861db8 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -2,7 +2,8 @@ import attr import marshmallow.fields -import pytypes +# import pytypes +import typeguard T = typing.TypeVar("T") @@ -124,8 +125,17 @@ def register( def from_object(self, value: typing.Any) -> HintTagField: for type_tag_field in self.the_list: - if pytypes.is_of_type(value, type_tag_field.hint): - return type_tag_field + # if pytypes.is_of_type(value, type_tag_field.hint): + try: + typeguard.check_type( + argname='', + value=value, + expected_type=type_tag_field.hint, + ) + except TypeError: + continue + + return type_tag_field raise Exception() diff --git a/test-requirements.txt b/test-requirements.txt index babe77d..b77a5e6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,7 +13,8 @@ argh==0.26.2 \ # via watchdog attrs==19.3.0 \ --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \ - --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 + --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 \ + # via -r requirements.in, pytest click==7.0 \ --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ @@ -49,31 +50,38 @@ coverage==5.0.3 \ --hash=sha256:da93027835164b8223e8e5af2cf902a4c80ed93cb0909417234f4a9df3bcd9af \ --hash=sha256:e69215621707119c6baf99bda014a45b999d37602cb7043d943c76a59b05bf52 \ --hash=sha256:ea9525e0fef2de9208250d6c5aeeee0138921057cd67fcef90fbed49c4d62d37 \ - --hash=sha256:fca1669d464f0c9831fd10be2eef6b86f5ebd76c724d1e0706ebdff86bb4adf0 + --hash=sha256:fca1669d464f0c9831fd10be2eef6b86f5ebd76c724d1e0706ebdff86bb4adf0 \ + # via -r test-requirements.in, cuvner, pytest-cov cuvner==18.0.1 \ - --hash=sha256:395b9b1d802aca999212e70566813b79c6a50289d6a2a712252c9c9eb52cf29e + --hash=sha256:395b9b1d802aca999212e70566813b79c6a50289d6a2a712252c9c9eb52cf29e \ + # via -r test-requirements.in dataclasses==0.6 \ --hash=sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f \ - --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84 + --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84 \ + # via -r requirements.in filelock==3.0.12 \ --hash=sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59 \ --hash=sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836 \ # via tox importlib-metadata==1.4.0 \ --hash=sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359 \ - --hash=sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8 + --hash=sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8 \ + # via -r test-requirements.in incremental==17.5.0 \ --hash=sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f \ --hash=sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3 \ # via cuvner marshmallow-enum==1.5.1 \ --hash=sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58 \ - --hash=sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072 + --hash=sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072 \ + # via -r test-requirements.in marshmallow-union==0.1.12 \ - --hash=sha256:280708177aba5c2bb69614f7f7603c8a8fd265f6733890c6499fcd72bc27c1e7 + --hash=sha256:280708177aba5c2bb69614f7f7603c8a8fd265f6733890c6499fcd72bc27c1e7 \ + # via -r test-requirements.in marshmallow==3.3.0 \ --hash=sha256:0ba81b6da4ae69eb229b74b3c741ff13fe04fb899824377b1aff5aaa1a9fd46e \ - --hash=sha256:3e53dd9e9358977a3929e45cdbe4a671f9eff53a7d6a23f33ed3eab8c1890d8f + --hash=sha256:3e53dd9e9358977a3929e45cdbe4a671f9eff53a7d6a23f33ed3eab8c1890d8f \ + # via -r requirements.in, marshmallow-enum, marshmallow-union more-itertools==8.1.0 \ --hash=sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39 \ --hash=sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288 \ @@ -107,15 +115,19 @@ pyparsing==2.4.6 \ # via packaging pytest-cov==2.8.1 \ --hash=sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b \ - --hash=sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626 + --hash=sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626 \ + # via -r test-requirements.in pytest-sphinx==0.2.2 \ - --hash=sha256:3f2f55af893f717c9befc5aa6dbcaeda3174258706dfd3dffb4cfe9a1671f8ed + --hash=sha256:3f2f55af893f717c9befc5aa6dbcaeda3174258706dfd3dffb4cfe9a1671f8ed \ + # via -r test-requirements.in pytest-travis-fold==1.3.0 \ --hash=sha256:3fe15aa21ed14275e5a77814339281b3b618e350b98a43e7ac5d5bdcad8202cb \ - --hash=sha256:5607df571232b257be644400be559afb9148af3a27576f8080f56cee915771b2 + --hash=sha256:5607df571232b257be644400be559afb9148af3a27576f8080f56cee915771b2 \ + # via -r test-requirements.in pytest==5.3.4 \ --hash=sha256:1d122e8be54d1a709e56f82e2d85dcba3018313d64647f38a91aec88c239b600 \ - --hash=sha256:c13d1943c63e599b98cf118fcb9703e4d7bde7caa9a432567bcdcae4bf512d20 + --hash=sha256:c13d1943c63e599b98cf118fcb9703e4d7bde7caa9a432567bcdcae4bf512d20 \ + # via -r test-requirements.in, pytest-cov, pytest-sphinx, pytest-travis-fold pyyaml==5.3 \ --hash=sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6 \ --hash=sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf \ @@ -139,7 +151,12 @@ toml==0.10.0 \ # via tox tox==3.14.3 \ --hash=sha256:06ba73b149bf838d5cd25dc30c2dd2671ae5b2757cf98e5c41a35fe449f131b3 \ - --hash=sha256:806d0a9217584558cc93747a945a9d9bff10b141a5287f0c8429a08828a22192 + --hash=sha256:806d0a9217584558cc93747a945a9d9bff10b141a5287f0c8429a08828a22192 \ + # via -r test-requirements.in +typeguard==2.7.1 \ + --hash=sha256:1d3710251d3d3d6c64e0c49f45edec2e88ddc386a51e89c3ec0703efeb8b3b81 \ + --hash=sha256:2d545c71e9439c21bcd7c28f5f55b3606e6106f7031ab58375656a1aed483ef2 \ + # via -r requirements.in typing-extensions==3.7.4.1 \ --hash=sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2 \ --hash=sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d \ @@ -148,7 +165,8 @@ typing-extensions==3.7.4.1 \ typing-inspect==0.5.0 \ --hash=sha256:75c97b7854426a129f3184c68588db29091ff58e6908ed520add1d52fc44df6e \ --hash=sha256:811b44f92e780b90cfe7bac94249a4fae87cfaa9b40312765489255045231d9c \ - --hash=sha256:c6ed1cd34860857c53c146a6704a96da12e1661087828ce350f34addc6e5eee3 + --hash=sha256:c6ed1cd34860857c53c146a6704a96da12e1661087828ce350f34addc6e5eee3 \ + # via -r requirements.in unidiff==0.5.5 \ --hash=sha256:6e7ff4be1a9cd8d72197cd15ec735260b8b95d6f9d3e6fdc8a37301b12af0b27 \ --hash=sha256:9c9ab5fb96b6988b4cd5def6b275492442c04a570900d33aa6373105780025bc \ From 18e78d3af47df0db6f680df772dad78814c34fe8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 22:43:30 -0400 Subject: [PATCH 13/53] black --- src/desert/_fields.py | 25 ++++++------------------- tests/test_fields.py | 14 +++----------- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 3861db8..7df32cb 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -2,6 +2,7 @@ import attr import marshmallow.fields + # import pytypes import typeguard @@ -26,18 +27,12 @@ def from_tag(self, tag: str) -> marshmallow.fields.Field: @attr.s(auto_attribs=True) class TypeDictFieldRegistry: - the_dict: typing.Dict[ - typing.Union[type, str], - marshmallow.fields.Field, - ] = attr.ib( + the_dict: typing.Dict[typing.Union[type, str], marshmallow.fields.Field,] = attr.ib( factory=dict ) def register( - self, - hint: typing.Any, - tag: str, - field: marshmallow.fields.Field, + self, hint: typing.Any, tag: str, field: marshmallow.fields.Field, ) -> None: # TODO: just disabling for now to show more interesting test results # if any(key in self.the_dict for key in [cls, tag]): @@ -67,10 +62,7 @@ class OrderedIsinstanceFieldRegistry: # TODO: but type bans from-scratch metatypes... and protocols def register( - self, - hint: typing.Any, - tag: str, - field: marshmallow.fields.Field, + self, hint: typing.Any, tag: str, field: marshmallow.fields.Field, ) -> None: # TODO: just disabling for now to show more interesting test results # if any(key in self.the_dict for key in [cls, tag]): @@ -104,10 +96,7 @@ class OrderedIsinstanceFieldRegistry: # TODO: but type bans from-scratch metatypes... and protocols def register( - self, - hint: typing.Any, - tag: str, - field: marshmallow.fields.Field, + self, hint: typing.Any, tag: str, field: marshmallow.fields.Field, ) -> None: # TODO: just disabling for now to show more interesting test results # if any(key in self.the_dict for key in [cls, tag]): @@ -128,9 +117,7 @@ def from_object(self, value: typing.Any) -> HintTagField: # if pytypes.is_of_type(value, type_tag_field.hint): try: typeguard.check_type( - argname='', - value=value, - expected_type=type_tag_field.hint, + argname="", value=value, expected_type=type_tag_field.hint, ) except TypeError: continue diff --git a/tests/test_fields.py b/tests/test_fields.py index a045a06..f3bcdd0 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -47,16 +47,10 @@ def build( example_data_list = [ ExampleData.build( - hint=float, - to_serialize=3.7, - tag="float_tag", - field=marshmallow.fields.Float(), + hint=float, to_serialize=3.7, tag="float_tag", field=marshmallow.fields.Float(), ), ExampleData.build( - hint=str, - to_serialize="29", - tag="str_tag", - field=marshmallow.fields.String(), + hint=str, to_serialize="29", tag="str_tag", field=marshmallow.fields.String(), ), ExampleData.build( hint=decimal.Decimal, @@ -127,9 +121,7 @@ def build_order_isinstance_registry(examples): for example in examples: registry.register( - hint=example.hint, - tag=example.tag, - field=example.field, + hint=example.hint, tag=example.tag, field=example.field, ) return registry From b233cf47648e762e2ceee3b8f3fe4e493f592840 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 22:46:47 -0400 Subject: [PATCH 14/53] cleanup --- src/desert/_fields.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 7df32cb..1c5f181 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -2,8 +2,6 @@ import attr import marshmallow.fields - -# import pytypes import typeguard From 7a44b411a4207c790df5cae138c39613ed8335ec Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 22:57:33 -0400 Subject: [PATCH 15/53] no protocol for now --- src/desert/_fields.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 1c5f181..e5e3490 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -15,12 +15,12 @@ class HintTagField: field: marshmallow.fields.Field -class FieldRegistry(typing.Protocol): - def from_object(self, value: typing.Any) -> marshmallow.fields.Field: - ... - - def from_tag(self, tag: str) -> marshmallow.fields.Field: - ... +# class FieldRegistry(typing.Protocol): +# def from_object(self, value: typing.Any) -> marshmallow.fields.Field: +# ... +# +# def from_tag(self, tag: str) -> marshmallow.fields.Field: +# ... @attr.s(auto_attribs=True) From 7731c00ab0be512adb7a04a7bbb48d4bf57e646c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 22:59:40 -0400 Subject: [PATCH 16/53] remove duplicate code --- src/desert/_fields.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index e5e3490..add31ba 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -53,40 +53,6 @@ def from_tag(self, tag: str) -> marshmallow.fields.Field: return self.the_dict[tag] -@attr.s(auto_attribs=True) -class OrderedIsinstanceFieldRegistry: - the_list: typing.List[HintTagField] = attr.ib(factory=list) - by_tag: typing.Dict[str, HintTagField] = attr.ib(factory=dict) - - # TODO: but type bans from-scratch metatypes... and protocols - def register( - self, hint: typing.Any, tag: str, field: marshmallow.fields.Field, - ) -> None: - # TODO: just disabling for now to show more interesting test results - # if any(key in self.the_dict for key in [cls, tag]): - # raise Exception() - - type_tag_field = HintTagField(hint=hint, tag=tag, field=field) - - self.the_list.append(type_tag_field) - self.by_tag[tag] = type_tag_field - - # # TODO: this type hinting... doesn't help much as it could return - # # another cls - # def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: - # return lambda cls: self.register(cls=cls, tag=tag, field=field) - - def from_object(self, value: typing.Any) -> HintTagField: - for type_tag_field in self.the_list: - if isinstance(value, type_tag_field.cls): - return type_tag_field - - raise Exception() - - def from_tag(self, tag: str) -> HintTagField: - return self.by_tag[tag] - - @attr.s(auto_attribs=True) class OrderedIsinstanceFieldRegistry: the_list: typing.List[HintTagField] = attr.ib(factory=list) From b30170e3713abb5fa82d9c3904729897be74bd34 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 23:01:56 -0400 Subject: [PATCH 17/53] pop a little --- src/desert/_fields.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index add31ba..86e02fd 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -114,10 +114,10 @@ def _deserialize( data: typing.Optional[typing.Mapping[str, typing.Any]], **kwargs, ) -> typing.Any: - tag = value["type"] - serialized_value = value["value"] + tag = value.pop("type") + serialized_value = value.pop("value") - if len(value) > 2: + if len(value) > 0: raise Exception() type_tag_field = self.from_tag(tag) From 21c9e1b1402ed5b5d0eeecaae5f1669864205354 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 23:17:06 -0400 Subject: [PATCH 18/53] extract adjacent functions and generalize to TaggedUnion --- src/desert/_fields.py | 49 +++++++++++++++++++++++++++++++++++-------- tests/test_fields.py | 2 +- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 86e02fd..1165110 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -1,3 +1,4 @@ +import functools import typing import attr @@ -94,18 +95,42 @@ def from_tag(self, tag: str) -> HintTagField: return self.by_tag[tag] -class AdjacentlyTaggedUnion(marshmallow.fields.Field): +@attr.s(auto_attribs=True) +class TaggedValue: + tag: str + value: typing.Any + + +def from_adjacently_tagged(item: typing.Any): + tag = item.pop("type") + serialized_value = item.pop("value") + + if len(item) > 0: + raise Exception() + + return TaggedValue(tag=tag, value=serialized_value) + + +def to_adjacently_tagged(tag: str, value: typing.Any): + return {"type": tag, "value": value} + + +class TaggedUnion(marshmallow.fields.Field): def __init__( self, *, from_object: typing.Callable[[typing.Any], HintTagField], from_tag: typing.Callable[[str], HintTagField], + from_tagged: typing.Callable[[typing.Any], typing.Any], + to_tagged: typing.Callable[[str, typing.Any], TaggedValue], **kwargs, ): super().__init__(**kwargs) self.from_object = from_object self.from_tag = from_tag + self.from_tagged = from_tagged + self.to_tagged = to_tagged def _deserialize( self, @@ -114,16 +139,12 @@ def _deserialize( data: typing.Optional[typing.Mapping[str, typing.Any]], **kwargs, ) -> typing.Any: - tag = value.pop("type") - serialized_value = value.pop("value") - - if len(value) > 0: - raise Exception() + tagged_value = self.from_tagged(item=value) - type_tag_field = self.from_tag(tag) + type_tag_field = self.from_tag(tagged_value.tag) field = type_tag_field.field - return field.deserialize(serialized_value) + return field.deserialize(tagged_value.value) def _serialize( self, value: typing.Any, attr: str, obj: typing.Any, **kwargs, @@ -133,4 +154,14 @@ def _serialize( tag = type_tag_field.tag serialized_value = field.serialize(attr, obj) - return {"type": tag, "value": serialized_value} + return self.to_tagged(tag=tag, value=serialized_value) + + +@functools.wraps(TaggedUnion) +def adjacently_tagged_union(*args, **kwargs): + return TaggedUnion( + *args, + from_tagged=from_adjacently_tagged, + to_tagged=to_adjacently_tagged, + **kwargs, + ) diff --git a/tests/test_fields.py b/tests/test_fields.py index f3bcdd0..aaecd49 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -168,7 +168,7 @@ def _registry(request): @pytest.fixture(name="adjacently_tagged_field") def _adjacently_tagged_field(registry): - return desert._fields.AdjacentlyTaggedUnion( + return desert._fields.adjacently_tagged_union( from_object=registry.from_object, from_tag=registry.from_tag, ) From 6a4bae8f8f753185a6804d43f0cf91af4a7bbaf2 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 23:18:44 -0400 Subject: [PATCH 19/53] group adjacently tagged functions --- src/desert/_fields.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 1165110..ae62216 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -101,20 +101,6 @@ class TaggedValue: value: typing.Any -def from_adjacently_tagged(item: typing.Any): - tag = item.pop("type") - serialized_value = item.pop("value") - - if len(item) > 0: - raise Exception() - - return TaggedValue(tag=tag, value=serialized_value) - - -def to_adjacently_tagged(tag: str, value: typing.Any): - return {"type": tag, "value": value} - - class TaggedUnion(marshmallow.fields.Field): def __init__( self, @@ -157,6 +143,20 @@ def _serialize( return self.to_tagged(tag=tag, value=serialized_value) +def from_adjacently_tagged(item: typing.Any): + tag = item.pop("type") + serialized_value = item.pop("value") + + if len(item) > 0: + raise Exception() + + return TaggedValue(tag=tag, value=serialized_value) + + +def to_adjacently_tagged(tag: str, value: typing.Any): + return {"type": tag, "value": value} + + @functools.wraps(TaggedUnion) def adjacently_tagged_union(*args, **kwargs): return TaggedUnion( From 8b97cf6a09dc84aa80c95212c8286576c5b64b19 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 23:23:07 -0400 Subject: [PATCH 20/53] add externally tagged support --- src/desert/_fields.py | 24 ++++++++++++++++++++++-- tests/test_fields.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index ae62216..f7d2dc3 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -107,8 +107,8 @@ def __init__( *, from_object: typing.Callable[[typing.Any], HintTagField], from_tag: typing.Callable[[str], HintTagField], - from_tagged: typing.Callable[[typing.Any], typing.Any], - to_tagged: typing.Callable[[str, typing.Any], TaggedValue], + from_tagged: typing.Callable[[typing.Any], TaggedValue], + to_tagged: typing.Callable[[str, typing.Any], typing.Any], **kwargs, ): super().__init__(**kwargs) @@ -143,6 +143,26 @@ def _serialize( return self.to_tagged(tag=tag, value=serialized_value) +def from_externally_tagged(item: typing.Any): + [[tag, serialized_value]] = item.items() + + return TaggedValue(tag=tag, value=serialized_value) + + +def to_externally_tagged(tag: str, value: typing.Any): + return {tag: value} + + +@functools.wraps(TaggedUnion) +def externally_tagged_union(*args, **kwargs): + return TaggedUnion( + *args, + from_tagged=from_externally_tagged, + to_tagged=to_externally_tagged, + **kwargs, + ) + + def from_adjacently_tagged(item: typing.Any): tag = item.pop("type") serialized_value = item.pop("value") diff --git a/tests/test_fields.py b/tests/test_fields.py index aaecd49..b7181c7 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -202,3 +202,42 @@ def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): serialized = adjacently_tagged_field.serialize("key", obj) assert serialized == {"type": example_data.tag, "value": example_data.serialized} + + +@pytest.fixture(name="externally_tagged_field") +def _externally_tagged_field(registry): + return desert._fields.externally_tagged_union( + from_object=registry.from_object, from_tag=registry.from_tag, + ) + + +def test_externally_tagged_deserialize(example_data, externally_tagged_field): + serialized = {example_data.tag: example_data.serialized} + + deserialized = externally_tagged_field.deserialize(serialized) + + expected = example_data.deserialized + + assert (type(deserialized) == type(expected)) and (deserialized == expected) + + +def test_externally_tagged_deserialize_extra_key_raises( + example_data, externally_tagged_field, +): + serialized = { + example_data.tag: { + "value": example_data.serialized, + "extra": 29, + }, + } + + with pytest.raises(expected_exception=Exception): + externally_tagged_field.deserialize(serialized) + + +def test_externally_tagged_serialize(example_data, externally_tagged_field): + obj = {"key": example_data.to_serialize} + + serialized = externally_tagged_field.serialize("key", obj) + + assert serialized == {example_data.tag: example_data.serialized} From f63f290198ec86caea27b690efa8ad6b745369a8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 May 2020 23:38:34 -0400 Subject: [PATCH 21/53] add internally tagged support --- src/desert/_fields.py | 25 +++++++++ tests/test_fields.py | 124 ++++++++++++++++++++++++++++++------------ 2 files changed, 114 insertions(+), 35 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index f7d2dc3..9761a33 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -163,6 +163,31 @@ def externally_tagged_union(*args, **kwargs): ) +def from_internally_tagged(item: typing.Any): + # TODO: shouldn't be hardcoded to "type" + return TaggedValue( + tag=item["type"], value={k: v for k, v in item.items() if k != "type"} + ) + + +def to_internally_tagged(tag: str, value: typing.Any): + # TODO: shouldn't be hardcoded to "type" + if "type" in value: + raise Exception() + + return {"type": tag, **value} + + +@functools.wraps(TaggedUnion) +def internally_tagged_union(*args, **kwargs): + return TaggedUnion( + *args, + from_tagged=from_internally_tagged, + to_tagged=to_internally_tagged, + **kwargs, + ) + + def from_adjacently_tagged(item: typing.Any): tag = item.pop("type") serialized_value = item.pop("value") diff --git a/tests/test_fields.py b/tests/test_fields.py index b7181c7..446385e 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -45,7 +45,7 @@ def build( ) -example_data_list = [ +basic_example_data_list = [ ExampleData.build( hint=float, to_serialize=3.7, tag="float_tag", field=marshmallow.fields.Float(), ), @@ -82,15 +82,44 @@ def build( ] +@attr.s(auto_attribs=True) +class CustomExampleClass: + a: int + b: str + + +custom_example_data_list = [ + ExampleData.build( + hint=CustomExampleClass, + to_serialize=CustomExampleClass(a=1, b="b"), + serialized={"a": 1, "b": "b"}, + tag="custom_example_class", + field=marshmallow.fields.Nested(desert.schema(CustomExampleClass)), + ), +] + + +all_example_data_list = basic_example_data_list + custom_example_data_list + + @pytest.fixture( name="example_data", - params=example_data_list, - ids=[str(example) for example in example_data_list], + params=all_example_data_list, + ids=[str(example) for example in all_example_data_list], ) def _example_data(request): return request.param +@pytest.fixture( + name="custom_example_data", + params=custom_example_data_list, + ids=[str(example) for example in custom_example_data_list], +) +def _custom_example_data(request): + return request.param + + def build_type_dict_registry(examples): registry = desert._fields.TypeDictFieldRegistry() @@ -154,7 +183,7 @@ def build_order_isinstance_registry(examples): registries = [ # build_type_dict_registry(example_data_list), - build_order_isinstance_registry(example_data_list), + build_order_isinstance_registry(all_example_data_list), ] registry_ids = [type(registry).__name__ for registry in registries] @@ -166,78 +195,103 @@ def _registry(request): return request.param -@pytest.fixture(name="adjacently_tagged_field") -def _adjacently_tagged_field(registry): - return desert._fields.adjacently_tagged_union( +@pytest.fixture(name="externally_tagged_field") +def _externally_tagged_field(registry): + return desert._fields.externally_tagged_union( from_object=registry.from_object, from_tag=registry.from_tag, ) -def test_adjacently_tagged_deserialize(example_data, adjacently_tagged_field): - serialized = {"type": example_data.tag, "value": example_data.serialized} +def test_externally_tagged_deserialize(example_data, externally_tagged_field): + serialized = {example_data.tag: example_data.serialized} - deserialized = adjacently_tagged_field.deserialize(serialized) + deserialized = externally_tagged_field.deserialize(serialized) expected = example_data.deserialized assert (type(deserialized) == type(expected)) and (deserialized == expected) -def test_adjacently_tagged_deserialize_extra_key_raises( - example_data, adjacently_tagged_field, +def test_externally_tagged_deserialize_extra_key_raises( + example_data, externally_tagged_field, ): serialized = { - "type": example_data.tag, - "value": example_data.serialized, - "extra": 29, + example_data.tag: {"value": example_data.serialized, "extra": 29,}, } with pytest.raises(expected_exception=Exception): - adjacently_tagged_field.deserialize(serialized) + externally_tagged_field.deserialize(serialized) -def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): +def test_externally_tagged_serialize(example_data, externally_tagged_field): obj = {"key": example_data.to_serialize} - serialized = adjacently_tagged_field.serialize("key", obj) + serialized = externally_tagged_field.serialize("key", obj) - assert serialized == {"type": example_data.tag, "value": example_data.serialized} + assert serialized == {example_data.tag: example_data.serialized} -@pytest.fixture(name="externally_tagged_field") -def _externally_tagged_field(registry): - return desert._fields.externally_tagged_union( +@pytest.fixture(name="internally_tagged_field") +def _internally_tagged_field(registry): + return desert._fields.internally_tagged_union( from_object=registry.from_object, from_tag=registry.from_tag, ) -def test_externally_tagged_deserialize(example_data, externally_tagged_field): - serialized = {example_data.tag: example_data.serialized} +def test_internally_tagged_deserialize(custom_example_data, internally_tagged_field): + serialized = {"type": custom_example_data.tag, **custom_example_data.serialized} - deserialized = externally_tagged_field.deserialize(serialized) + deserialized = internally_tagged_field.deserialize(serialized) + + expected = custom_example_data.deserialized + + assert (type(deserialized) == type(expected)) and (deserialized == expected) + + +def test_internally_tagged_serialize(custom_example_data, internally_tagged_field): + obj = {"key": custom_example_data.to_serialize} + + serialized = internally_tagged_field.serialize("key", obj) + + assert serialized == { + "type": custom_example_data.tag, + **custom_example_data.serialized, + } + + +@pytest.fixture(name="adjacently_tagged_field") +def _adjacently_tagged_field(registry): + return desert._fields.adjacently_tagged_union( + from_object=registry.from_object, from_tag=registry.from_tag, + ) + + +def test_adjacently_tagged_deserialize(example_data, adjacently_tagged_field): + serialized = {"type": example_data.tag, "value": example_data.serialized} + + deserialized = adjacently_tagged_field.deserialize(serialized) expected = example_data.deserialized assert (type(deserialized) == type(expected)) and (deserialized == expected) -def test_externally_tagged_deserialize_extra_key_raises( - example_data, externally_tagged_field, +def test_adjacently_tagged_deserialize_extra_key_raises( + example_data, adjacently_tagged_field, ): serialized = { - example_data.tag: { - "value": example_data.serialized, - "extra": 29, - }, + "type": example_data.tag, + "value": example_data.serialized, + "extra": 29, } with pytest.raises(expected_exception=Exception): - externally_tagged_field.deserialize(serialized) + adjacently_tagged_field.deserialize(serialized) -def test_externally_tagged_serialize(example_data, externally_tagged_field): +def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): obj = {"key": example_data.to_serialize} - serialized = externally_tagged_field.serialize("key", obj) + serialized = adjacently_tagged_field.serialize("key", obj) - assert serialized == {example_data.tag: example_data.serialized} + assert serialized == {"type": example_data.tag, "value": example_data.serialized} From 1b412172590eb8ad85dabdd506db5f2fd3149223 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 31 May 2020 19:54:51 -0400 Subject: [PATCH 22/53] make tagged type and value keys configurable --- src/desert/_fields.py | 47 ++++++++++++++++++++++++++----------------- tests/test_fields.py | 14 ++++++------- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 9761a33..f62a6de 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -143,6 +143,10 @@ def _serialize( return self.to_tagged(tag=tag, value=serialized_value) +default_tagged_type_key = "#type" +default_tagged_value_key = "#value" + + def from_externally_tagged(item: typing.Any): [[tag, serialized_value]] = item.items() @@ -163,34 +167,32 @@ def externally_tagged_union(*args, **kwargs): ) -def from_internally_tagged(item: typing.Any): - # TODO: shouldn't be hardcoded to "type" +def from_internally_tagged(item: typing.Any, type_key: str): return TaggedValue( - tag=item["type"], value={k: v for k, v in item.items() if k != "type"} + tag=item[type_key], value={k: v for k, v in item.items() if k != type_key} ) -def to_internally_tagged(tag: str, value: typing.Any): - # TODO: shouldn't be hardcoded to "type" - if "type" in value: +def to_internally_tagged(tag: str, value: typing.Any, type_key: str): + if type_key in value: raise Exception() - return {"type": tag, **value} + return {type_key: tag, **value} @functools.wraps(TaggedUnion) -def internally_tagged_union(*args, **kwargs): +def internally_tagged_union(*args, type_key=default_tagged_type_key, **kwargs): return TaggedUnion( *args, - from_tagged=from_internally_tagged, - to_tagged=to_internally_tagged, + from_tagged=functools.partial(from_internally_tagged, type_key=type_key), + to_tagged=functools.partial(to_internally_tagged, type_key=type_key), **kwargs, ) -def from_adjacently_tagged(item: typing.Any): - tag = item.pop("type") - serialized_value = item.pop("value") +def from_adjacently_tagged(item: typing.Any, type_key: str, value_key: str): + tag = item.pop(type_key) + serialized_value = item.pop(value_key) if len(item) > 0: raise Exception() @@ -198,15 +200,24 @@ def from_adjacently_tagged(item: typing.Any): return TaggedValue(tag=tag, value=serialized_value) -def to_adjacently_tagged(tag: str, value: typing.Any): - return {"type": tag, "value": value} +def to_adjacently_tagged(tag: str, value: typing.Any, type_key: str, value_key: str): + return {type_key: tag, value_key: value} @functools.wraps(TaggedUnion) -def adjacently_tagged_union(*args, **kwargs): +def adjacently_tagged_union( + *args, + type_key=default_tagged_type_key, + value_key=default_tagged_value_key, + **kwargs, +): return TaggedUnion( *args, - from_tagged=from_adjacently_tagged, - to_tagged=to_adjacently_tagged, + from_tagged=functools.partial( + from_adjacently_tagged, type_key=type_key, value_key=value_key + ), + to_tagged=functools.partial( + to_adjacently_tagged, type_key=type_key, value_key=value_key + ), **kwargs, ) diff --git a/tests/test_fields.py b/tests/test_fields.py index 446385e..8179ccd 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -216,7 +216,7 @@ def test_externally_tagged_deserialize_extra_key_raises( example_data, externally_tagged_field, ): serialized = { - example_data.tag: {"value": example_data.serialized, "extra": 29,}, + example_data.tag: {"#value": example_data.serialized, "extra": 29,}, } with pytest.raises(expected_exception=Exception): @@ -239,7 +239,7 @@ def _internally_tagged_field(registry): def test_internally_tagged_deserialize(custom_example_data, internally_tagged_field): - serialized = {"type": custom_example_data.tag, **custom_example_data.serialized} + serialized = {"#type": custom_example_data.tag, **custom_example_data.serialized} deserialized = internally_tagged_field.deserialize(serialized) @@ -254,7 +254,7 @@ def test_internally_tagged_serialize(custom_example_data, internally_tagged_fiel serialized = internally_tagged_field.serialize("key", obj) assert serialized == { - "type": custom_example_data.tag, + "#type": custom_example_data.tag, **custom_example_data.serialized, } @@ -267,7 +267,7 @@ def _adjacently_tagged_field(registry): def test_adjacently_tagged_deserialize(example_data, adjacently_tagged_field): - serialized = {"type": example_data.tag, "value": example_data.serialized} + serialized = {"#type": example_data.tag, "#value": example_data.serialized} deserialized = adjacently_tagged_field.deserialize(serialized) @@ -280,8 +280,8 @@ def test_adjacently_tagged_deserialize_extra_key_raises( example_data, adjacently_tagged_field, ): serialized = { - "type": example_data.tag, - "value": example_data.serialized, + "#type": example_data.tag, + "#value": example_data.serialized, "extra": 29, } @@ -294,4 +294,4 @@ def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): serialized = adjacently_tagged_field.serialize("key", obj) - assert serialized == {"type": example_data.tag, "value": example_data.serialized} + assert serialized == {"#type": example_data.tag, "#value": example_data.serialized} From 74494eb1dda4e6df90e76dfd6ff37f969a9f0a9b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 1 Jun 2020 13:27:00 -0400 Subject: [PATCH 23/53] complain if other than exactly one hint matches --- src/desert/_fields.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index f62a6de..fa8e8d9 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -78,6 +78,8 @@ def register( # return lambda cls: self.register(cls=cls, tag=tag, field=field) def from_object(self, value: typing.Any) -> HintTagField: + potential = set() + for type_tag_field in self.the_list: # if pytypes.is_of_type(value, type_tag_field.hint): try: @@ -87,9 +89,18 @@ def from_object(self, value: typing.Any) -> HintTagField: except TypeError: continue - return type_tag_field + potential.add(type_tag_field) - raise Exception() + if len(potential) != 1: + raise Exception( + "Unique matching type hint not found: {}".format( + {p.hint for p in potential}, + ) + ) + + [type_tag_field] = potential + + return type_tag_field def from_tag(self, tag: str) -> HintTagField: return self.by_tag[tag] From d5d7183e3e5c497034289dc9c701020f657108b5 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 1 Jun 2020 14:23:50 -0400 Subject: [PATCH 24/53] isinstance() for str vs. typing.Sequence[str] handling --- src/desert/_fields.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index fa8e8d9..4355ea9 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -1,3 +1,4 @@ +import collections import functools import typing @@ -78,23 +79,44 @@ def register( # return lambda cls: self.register(cls=cls, tag=tag, field=field) def from_object(self, value: typing.Any) -> HintTagField: - potential = set() + scores = {} for type_tag_field in self.the_list: + score = 0 + # if pytypes.is_of_type(value, type_tag_field.hint): try: typeguard.check_type( argname="", value=value, expected_type=type_tag_field.hint, ) except TypeError: - continue + pass + else: + score = max(1, score) + + try: + if isinstance(value, type_tag_field.hint): + score = max(2, score) + except TypeError: + pass + + scores[type_tag_field] = score + + high_score = max(scores.values()) + + if high_score == 0: + raise Exception("No matching type hints found") - potential.add(type_tag_field) + potential = [ + ttf + for ttf, score in scores.items() + if score == high_score + ] if len(potential) != 1: raise Exception( "Unique matching type hint not found: {}".format( - {p.hint for p in potential}, + ', '.join(str(p.hint) for p in potential), ) ) From f29ef21c1b30d83b80afab6c3f4e09cf28003f6c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 29 Jul 2021 14:38:46 -0400 Subject: [PATCH 25/53] black --- src/desert/_fields.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 4355ea9..5e5efc5 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -27,12 +27,16 @@ class HintTagField: @attr.s(auto_attribs=True) class TypeDictFieldRegistry: - the_dict: typing.Dict[typing.Union[type, str], marshmallow.fields.Field,] = attr.ib( - factory=dict - ) + the_dict: typing.Dict[ + typing.Union[type, str], + marshmallow.fields.Field, + ] = attr.ib(factory=dict) def register( - self, hint: typing.Any, tag: str, field: marshmallow.fields.Field, + self, + hint: typing.Any, + tag: str, + field: marshmallow.fields.Field, ) -> None: # TODO: just disabling for now to show more interesting test results # if any(key in self.the_dict for key in [cls, tag]): @@ -62,7 +66,10 @@ class OrderedIsinstanceFieldRegistry: # TODO: but type bans from-scratch metatypes... and protocols def register( - self, hint: typing.Any, tag: str, field: marshmallow.fields.Field, + self, + hint: typing.Any, + tag: str, + field: marshmallow.fields.Field, ) -> None: # TODO: just disabling for now to show more interesting test results # if any(key in self.the_dict for key in [cls, tag]): @@ -87,7 +94,9 @@ def from_object(self, value: typing.Any) -> HintTagField: # if pytypes.is_of_type(value, type_tag_field.hint): try: typeguard.check_type( - argname="", value=value, expected_type=type_tag_field.hint, + argname="", + value=value, + expected_type=type_tag_field.hint, ) except TypeError: pass @@ -107,16 +116,12 @@ def from_object(self, value: typing.Any) -> HintTagField: if high_score == 0: raise Exception("No matching type hints found") - potential = [ - ttf - for ttf, score in scores.items() - if score == high_score - ] + potential = [ttf for ttf, score in scores.items() if score == high_score] if len(potential) != 1: raise Exception( "Unique matching type hint not found: {}".format( - ', '.join(str(p.hint) for p in potential), + ", ".join(str(p.hint) for p in potential), ) ) @@ -166,7 +171,11 @@ def _deserialize( return field.deserialize(tagged_value.value) def _serialize( - self, value: typing.Any, attr: str, obj: typing.Any, **kwargs, + self, + value: typing.Any, + attr: str, + obj: typing.Any, + **kwargs, ) -> typing.Any: type_tag_field = self.from_object(value) field = type_tag_field.field From d76ec24bcd20cdbd9bf12e0bd705041bd58f7ba4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 5 Aug 2021 10:37:32 -0400 Subject: [PATCH 26/53] additional heuristics for List vs. Sequence, 3.7+ only --- src/desert/_fields.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 5e5efc5..4d6df40 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -101,14 +101,24 @@ def from_object(self, value: typing.Any) -> HintTagField: except TypeError: pass else: - score = max(1, score) + score += 2 try: if isinstance(value, type_tag_field.hint): - score = max(2, score) + score += 3 except TypeError: pass + if score > 0: + # Only use this to disambiguate between already selected options such + # as ["a", "b"] matching both typing.List[str] and typing.Sequence[str]. + # This only works properly on 3.7+. + try: + if type(value) == type_tag_field.hint.__origin__: + score += 1 + except (AttributeError, TypeError): + pass + scores[type_tag_field] = score high_score = max(scores.values()) From 39caf6fc2155873bbd407e5f3dcbdb2b101cc716 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 5 Aug 2021 14:44:15 -0400 Subject: [PATCH 27/53] make test examples frozen --- tests/test_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index 8179ccd..7f61948 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -15,7 +15,7 @@ _NOTHING = object() -@attr.s(auto_attribs=True) +@attr.frozen class ExampleData: to_serialize: typing.Any serialized: typing.Any @@ -82,7 +82,7 @@ def build( ] -@attr.s(auto_attribs=True) +@attr.frozen class CustomExampleClass: a: int b: str From e61538033aefcd0c7ce9eb0f80cba1265c073e8b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 5 Aug 2021 15:40:32 -0400 Subject: [PATCH 28/53] mypy --- src/desert/_fields.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 4d6df40..9458429 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -29,7 +29,7 @@ class HintTagField: class TypeDictFieldRegistry: the_dict: typing.Dict[ typing.Union[type, str], - marshmallow.fields.Field, + HintTagField, ] = attr.ib(factory=dict) def register( @@ -52,10 +52,10 @@ def register( # def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: # return lambda cls: self.register(cls=cls, tag=tag, field=field) - def from_object(self, value: typing.Any) -> marshmallow.fields.Field: + def from_object(self, value: typing.Any) -> HintTagField: return self.the_dict[type(value)] - def from_tag(self, tag: str) -> marshmallow.fields.Field: + def from_tag(self, tag: str) -> HintTagField: return self.the_dict[tag] @@ -149,14 +149,34 @@ class TaggedValue: value: typing.Any +class FromObjectProtocol(typing.Protocol): + def __call__(self, value: object) -> HintTagField: + ... + + +class FromTagProtocol(typing.Protocol): + def __call__(self, tag: str) -> HintTagField: + ... + + +class FromTaggedProtocol(typing.Protocol): + def __call__(self, item: object) -> TaggedValue: + ... + + +class ToTaggedProtocol(typing.Protocol): + def __call__(self, tag: object, value: object) -> object: + ... + + class TaggedUnion(marshmallow.fields.Field): def __init__( self, *, - from_object: typing.Callable[[typing.Any], HintTagField], - from_tag: typing.Callable[[str], HintTagField], - from_tagged: typing.Callable[[typing.Any], TaggedValue], - to_tagged: typing.Callable[[str, typing.Any], typing.Any], + from_object: FromObjectProtocol, + from_tag: FromTagProtocol, + from_tagged: FromTaggedProtocol, + to_tagged: ToTaggedProtocol, **kwargs, ): super().__init__(**kwargs) From 0e9c3a93861e3a8c3c409ef5177fb68d80efd9aa Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 12:36:20 -0400 Subject: [PATCH 29/53] actually use typing-extensions --- src/desert/_fields.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 9458429..4aea6b5 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -5,7 +5,7 @@ import attr import marshmallow.fields import typeguard - +import typing_extensions T = typing.TypeVar("T") @@ -17,7 +17,7 @@ class HintTagField: field: marshmallow.fields.Field -# class FieldRegistry(typing.Protocol): +# class FieldRegistry(typing_extensions.Protocol): # def from_object(self, value: typing.Any) -> marshmallow.fields.Field: # ... # @@ -149,22 +149,22 @@ class TaggedValue: value: typing.Any -class FromObjectProtocol(typing.Protocol): +class FromObjectProtocol(typing_extensions.Protocol): def __call__(self, value: object) -> HintTagField: ... -class FromTagProtocol(typing.Protocol): +class FromTagProtocol(typing_extensions.Protocol): def __call__(self, tag: str) -> HintTagField: ... -class FromTaggedProtocol(typing.Protocol): +class FromTaggedProtocol(typing_extensions.Protocol): def __call__(self, item: object) -> TaggedValue: ... -class ToTaggedProtocol(typing.Protocol): +class ToTaggedProtocol(typing_extensions.Protocol): def __call__(self, tag: object, value: object) -> object: ... From 419fd30c77689a6f84ba8621e6107217388d027f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 12:38:23 -0400 Subject: [PATCH 30/53] black --- tests/test_fields.py | 54 +++++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index 7f61948..273e2a5 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -27,7 +27,13 @@ class ExampleData: @classmethod def build( - cls, hint, to_serialize, tag, field, serialized=_NOTHING, deserialized=_NOTHING, + cls, + hint, + to_serialize, + tag, + field, + serialized=_NOTHING, + deserialized=_NOTHING, ): if serialized is _NOTHING: serialized = to_serialize @@ -47,10 +53,16 @@ def build( basic_example_data_list = [ ExampleData.build( - hint=float, to_serialize=3.7, tag="float_tag", field=marshmallow.fields.Float(), + hint=float, + to_serialize=3.7, + tag="float_tag", + field=marshmallow.fields.Float(), ), ExampleData.build( - hint=str, to_serialize="29", tag="str_tag", field=marshmallow.fields.String(), + hint=str, + to_serialize="29", + tag="str_tag", + field=marshmallow.fields.String(), ), ExampleData.build( hint=decimal.Decimal, @@ -125,7 +137,9 @@ def build_type_dict_registry(examples): for example in examples: registry.register( - hint=example.hint, tag=example.tag, field=example.field, + hint=example.hint, + tag=example.tag, + field=example.field, ) return registry @@ -150,7 +164,9 @@ def build_order_isinstance_registry(examples): for example in examples: registry.register( - hint=example.hint, tag=example.tag, field=example.field, + hint=example.hint, + tag=example.tag, + field=example.field, ) return registry @@ -175,7 +191,9 @@ def build_order_isinstance_registry(examples): for example in examples: registry.register( - hint=example.hint, tag=example.tag, field=example.field, + hint=example.hint, + tag=example.tag, + field=example.field, ) return registry @@ -189,7 +207,9 @@ def build_order_isinstance_registry(examples): @pytest.fixture( - name="registry", params=registries, ids=registry_ids, + name="registry", + params=registries, + ids=registry_ids, ) def _registry(request): return request.param @@ -198,7 +218,8 @@ def _registry(request): @pytest.fixture(name="externally_tagged_field") def _externally_tagged_field(registry): return desert._fields.externally_tagged_union( - from_object=registry.from_object, from_tag=registry.from_tag, + from_object=registry.from_object, + from_tag=registry.from_tag, ) @@ -213,10 +234,14 @@ def test_externally_tagged_deserialize(example_data, externally_tagged_field): def test_externally_tagged_deserialize_extra_key_raises( - example_data, externally_tagged_field, + example_data, + externally_tagged_field, ): serialized = { - example_data.tag: {"#value": example_data.serialized, "extra": 29,}, + example_data.tag: { + "#value": example_data.serialized, + "extra": 29, + }, } with pytest.raises(expected_exception=Exception): @@ -234,7 +259,8 @@ def test_externally_tagged_serialize(example_data, externally_tagged_field): @pytest.fixture(name="internally_tagged_field") def _internally_tagged_field(registry): return desert._fields.internally_tagged_union( - from_object=registry.from_object, from_tag=registry.from_tag, + from_object=registry.from_object, + from_tag=registry.from_tag, ) @@ -262,7 +288,8 @@ def test_internally_tagged_serialize(custom_example_data, internally_tagged_fiel @pytest.fixture(name="adjacently_tagged_field") def _adjacently_tagged_field(registry): return desert._fields.adjacently_tagged_union( - from_object=registry.from_object, from_tag=registry.from_tag, + from_object=registry.from_object, + from_tag=registry.from_tag, ) @@ -277,7 +304,8 @@ def test_adjacently_tagged_deserialize(example_data, adjacently_tagged_field): def test_adjacently_tagged_deserialize_extra_key_raises( - example_data, adjacently_tagged_field, + example_data, + adjacently_tagged_field, ): serialized = { "#type": example_data.tag, From 7034ea2f1a0b9d5d80dcd89d90e846aa51f8b427 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 12:44:18 -0400 Subject: [PATCH 31/53] check --- src/desert/_fields.py | 1 + tests/test_fields.py | 27 --------------------------- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 4aea6b5..49f8672 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -7,6 +7,7 @@ import typeguard import typing_extensions + T = typing.TypeVar("T") diff --git a/tests/test_fields.py b/tests/test_fields.py index 273e2a5..6f5e9d6 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -172,33 +172,6 @@ def build_order_isinstance_registry(examples): return registry -# class NonStringSequence(abc.ABC): -# @classmethod -# def __subclasshook__(cls, maybe_subclass): -# return isinstance(maybe_subclass, collections.abc.Sequence) and not isinstance( -# maybe_subclass, str -# ) - - -def build_order_isinstance_registry(examples): - registry = desert._fields.OrderedIsinstanceFieldRegistry() - - # registry.register( - # hint=typing.Sequence, - # tag="sequence_abc", - # field=marshmallow.fields.List(marshmallow.fields.String()), - # ) - - for example in examples: - registry.register( - hint=example.hint, - tag=example.tag, - field=example.field, - ) - - return registry - - registries = [ # build_type_dict_registry(example_data_list), build_order_isinstance_registry(all_example_data_list), From fc889f64bb672d242adda41a169f8b1ee37d1ffc Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 13:35:17 -0400 Subject: [PATCH 32/53] coverage --- .coveragerc | 2 +- tests/test_fields.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index c9cf8c5..f621d2e 100644 --- a/.coveragerc +++ b/.coveragerc @@ -17,7 +17,7 @@ omit = *migrations* exclude_lines = # Lines matching these regexes don't need to be covered # https://coverage.readthedocs.io/en/coverage-5.5/excluding.html?highlight=exclude_lines#advanced-exclusion - + # this is the default but must be explicitly specified since # we are overriding exclude_lines pragma: no cover diff --git a/tests/test_fields.py b/tests/test_fields.py index 6f5e9d6..9f7c8fd 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -83,7 +83,7 @@ def build( tag="string_list_tag", field=marshmallow.fields.List(marshmallow.fields.String()), ), - ExampleData( + ExampleData.build( hint=typing.Sequence[str], to_serialize=("def", "13"), serialized=["def", "13"], From 408b21f9b55759f36a1a9cbb54fd942eb3eecb2e Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 14:46:20 -0400 Subject: [PATCH 33/53] create specific exceptions --- src/desert/_fields.py | 78 ++++++++++++++++++++-------------------- src/desert/exceptions.py | 32 +++++++++++++++++ tests/test_fields.py | 60 ++++++++++++++++++++++++------- 3 files changed, 120 insertions(+), 50 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 49f8672..2cf8284 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -7,6 +7,8 @@ import typeguard import typing_extensions +import desert.exceptions + T = typing.TypeVar("T") @@ -26,38 +28,38 @@ class HintTagField: # ... -@attr.s(auto_attribs=True) -class TypeDictFieldRegistry: - the_dict: typing.Dict[ - typing.Union[type, str], - HintTagField, - ] = attr.ib(factory=dict) - - def register( - self, - hint: typing.Any, - tag: str, - field: marshmallow.fields.Field, - ) -> None: - # TODO: just disabling for now to show more interesting test results - # if any(key in self.the_dict for key in [cls, tag]): - # raise Exception() - - type_tag_field = HintTagField(hint=hint, tag=tag, field=field) - - self.the_dict[hint] = type_tag_field - self.the_dict[tag] = type_tag_field - - # # TODO: this type hinting... doesn't help much as it could return - # # another cls - # def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: - # return lambda cls: self.register(cls=cls, tag=tag, field=field) - - def from_object(self, value: typing.Any) -> HintTagField: - return self.the_dict[type(value)] - - def from_tag(self, tag: str) -> HintTagField: - return self.the_dict[tag] +# @attr.s(auto_attribs=True) +# class TypeDictFieldRegistry: +# the_dict: typing.Dict[ +# typing.Union[type, str], +# HintTagField, +# ] = attr.ib(factory=dict) +# +# def register( +# self, +# hint: typing.Any, +# tag: str, +# field: marshmallow.fields.Field, +# ) -> None: +# # TODO: just disabling for now to show more interesting test results +# # if any(key in self.the_dict for key in [cls, tag]): +# # raise Exception() +# +# type_tag_field = HintTagField(hint=hint, tag=tag, field=field) +# +# self.the_dict[hint] = type_tag_field +# self.the_dict[tag] = type_tag_field +# +# # # TODO: this type hinting... doesn't help much as it could return +# # # another cls +# # def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: +# # return lambda cls: self.register(cls=cls, tag=tag, field=field) +# +# def from_object(self, value: typing.Any) -> HintTagField: +# return self.the_dict[type(value)] +# +# def from_tag(self, tag: str) -> HintTagField: +# return self.the_dict[tag] @attr.s(auto_attribs=True) @@ -125,15 +127,15 @@ def from_object(self, value: typing.Any) -> HintTagField: high_score = max(scores.values()) if high_score == 0: - raise Exception("No matching type hints found") + raise desert.exceptions.NoMatchingHintFound( + hints=[ttf.hint for ttf in self.the_list], value=value + ) potential = [ttf for ttf, score in scores.items() if score == high_score] if len(potential) != 1: - raise Exception( - "Unique matching type hint not found: {}".format( - ", ".join(str(p.hint) for p in potential), - ) + raise desert.exceptions.MultipleMatchingHintsFound( + hints=[ttf.hint for ttf in potential], value=value ) [type_tag_field] = potential @@ -248,7 +250,7 @@ def from_internally_tagged(item: typing.Any, type_key: str): def to_internally_tagged(tag: str, value: typing.Any, type_key: str): if type_key in value: - raise Exception() + raise desert.exceptions.TypeKeyCollision(type_key=type_key, value=value) return {type_key: tag, **value} diff --git a/src/desert/exceptions.py b/src/desert/exceptions.py index 3aac255..8bbb47c 100644 --- a/src/desert/exceptions.py +++ b/src/desert/exceptions.py @@ -1,4 +1,5 @@ import dataclasses +import typing as t import attr @@ -7,9 +8,40 @@ class DesertException(Exception): """Top-level exception for desert.""" +class MultipleMatchingHintsFound(DesertException): + """Raised when a union finds multiple hints that equally match the data to be + serialized. + """ + + def __init__(self, hints: t.Any, value: object): + hint_list = ", ".join(str(hint) for hint in hints) + super().__init__( + f"Multiple matching type hints found in union for {value!r}. Candidates: {hint_list}" + ) + + +class NoMatchingHintFound(DesertException): + """Raised when a union is unable to find a valid hint for the data to be + serialized. + """ + + def __init__(self, hints: t.Any, value: object): + hint_list = ", ".join(str(hint) for hint in hints) + super().__init__( + f"No matching type hints found in union for {value!r}. Considered: {hint_list}" + ) + + class NotAnAttrsClassOrDataclass(DesertException): """Raised for dataclass operations on non-dataclasses.""" +class TypeKeyCollision(DesertException): + """Raised when a tag key collides with a data value.""" + + def __init__(self, type_key: str, value: object): + super().__init__(f"Type key {type_key!r} collided with attribute in: {value!r}") + + class UnknownType(DesertException): """Raised for a type with unknown serialization equivalent.""" diff --git a/tests/test_fields.py b/tests/test_fields.py index 9f7c8fd..64db5f1 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -7,9 +7,9 @@ import marshmallow import pytest +import desert.exceptions import desert._fields - # TODO: test that field constructor doesn't tromple Field parameters _NOTHING = object() @@ -132,17 +132,17 @@ def _custom_example_data(request): return request.param -def build_type_dict_registry(examples): - registry = desert._fields.TypeDictFieldRegistry() - - for example in examples: - registry.register( - hint=example.hint, - tag=example.tag, - field=example.field, - ) - - return registry +# def build_type_dict_registry(examples): +# registry = desert._fields.TypeDictFieldRegistry() +# +# for example in examples: +# registry.register( +# hint=example.hint, +# tag=example.tag, +# field=example.field, +# ) +# +# return registry # class NonStringSequence(abc.ABC): @@ -188,6 +188,35 @@ def _registry(request): return request.param +def test_registry_raises_for_no_match(registry): + class C: + pass + + c = C() + + with pytest.raises(desert.exceptions.NoMatchingHintFound): + registry.from_object(value=c) + + +def test_registry_raises_for_multiple_matches(): + registry = desert._fields.OrderedIsinstanceFieldRegistry() + + registry.register( + hint=typing.Sequence, + tag="sequence", + field=marshmallow.fields.List(marshmallow.fields.Field()), + ) + + registry.register( + hint=typing.Collection, + tag="collection", + field=marshmallow.fields.List(marshmallow.fields.Field()), + ) + + with pytest.raises(desert.exceptions.MultipleMatchingHintsFound): + registry.from_object(value=[]) + + @pytest.fixture(name="externally_tagged_field") def _externally_tagged_field(registry): return desert._fields.externally_tagged_union( @@ -237,6 +266,13 @@ def _internally_tagged_field(registry): ) +def test_to_internally_tagged_raises_for_tag_collision(): + with pytest.raises(desert.exceptions.TypeKeyCollision): + desert._fields.to_internally_tagged( + tag="C", value={"collide": True}, type_key="collide" + ) + + def test_internally_tagged_deserialize(custom_example_data, internally_tagged_field): serialized = {"#type": custom_example_data.tag, **custom_example_data.serialized} From cc9279f48d05e61f234ae4ca4e8d5de4ad41e9e5 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 14:50:54 -0400 Subject: [PATCH 34/53] check --- tests/test_fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index 64db5f1..66e74e6 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -7,8 +7,9 @@ import marshmallow import pytest -import desert.exceptions import desert._fields +import desert.exceptions + # TODO: test that field constructor doesn't tromple Field parameters From a6dfccdf76f5fc0efbcac9146b77ebacb3e26b60 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 15:14:12 -0400 Subject: [PATCH 35/53] specify field registry protocol and check it --- src/desert/_fields.py | 25 +++++++++++++++++++------ src/desert/_util.py | 26 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 src/desert/_util.py diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 2cf8284..d4a24d5 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -7,6 +7,7 @@ import typeguard import typing_extensions +import desert._util import desert.exceptions @@ -20,12 +21,23 @@ class HintTagField: field: marshmallow.fields.Field -# class FieldRegistry(typing_extensions.Protocol): -# def from_object(self, value: typing.Any) -> marshmallow.fields.Field: -# ... -# -# def from_tag(self, tag: str) -> marshmallow.fields.Field: -# ... +class FieldRegistry(typing_extensions.Protocol): + def register( + self, + hint: typing.Any, + tag: str, + field: marshmallow.fields.Field, + ) -> None: + ... + + def from_object(self, value: typing.Any) -> HintTagField: + ... + + def from_tag(self, tag: str) -> HintTagField: + ... + + +check_field_registry_protocol = desert._util.ProtocolChecker[FieldRegistry]() # @attr.s(auto_attribs=True) @@ -62,6 +74,7 @@ class HintTagField: # return self.the_dict[tag] +@check_field_registry_protocol @attr.s(auto_attribs=True) class OrderedIsinstanceFieldRegistry: the_list: typing.List[HintTagField] = attr.ib(factory=list) diff --git a/src/desert/_util.py b/src/desert/_util.py new file mode 100644 index 0000000..73370b4 --- /dev/null +++ b/src/desert/_util.py @@ -0,0 +1,26 @@ +import typing + + +T = typing.TypeVar("T") + + +class ProtocolChecker(typing.Generic[T]): + """Instances of this class can be used as decorators that will result in type hint + checks to verifying that other classes implement a given protocol. Generally you + would create a single instance where you define each protocol and then use that + instance as the decorator. Note that this usage is, at least in part, due to + Python not supporting type parameter specification in the ``@`` decorator + expression. + .. code-block:: python + import typing + class MyProtocol(typing.Protocol): + def a_method(self): ... + check_my_protocol = qtrio._util.ProtocolChecker[MyProtocol]() + @check_my_protocol + class AClass: + def a_method(self): + return 42092 + """ + + def __call__(self, cls: typing.Type[T]) -> typing.Type[T]: + return cls From c7a385e4c19a2c0452d2b9f7d88291de7849f68a Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 17:33:38 -0400 Subject: [PATCH 36/53] add test that looks like real user code --- tests/test_fields.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_fields.py b/tests/test_fields.py index 66e74e6..9e80e1e 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,6 +1,7 @@ import abc import collections.abc import decimal +import json import typing import attr @@ -333,3 +334,27 @@ def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): serialized = adjacently_tagged_field.serialize("key", obj) assert serialized == {"#type": example_data.tag, "#value": example_data.serialized} + + +def test_actual_example(): + registry = desert._fields.OrderedIsinstanceFieldRegistry() + registry.register(hint=str, tag="str", field=marshmallow.fields.String()) + registry.register(hint=int, tag="int", field=marshmallow.fields.Integer()) + + field = desert._fields.adjacently_tagged_union( + from_object=registry.from_object, from_tag=registry.from_tag + ) + + @attr.frozen + class C: + # TODO: desert.ib() shouldn't be needed for many cases + union: typing.Union[str, int] = desert.ib(marshmallow_field=field) + + schema = desert.schema(C) + + objects = C(union="3") + marshalled = {"union": {"#type": "str", "#value": "3"}} + serialized = json.dumps(marshalled) + + assert schema.dumps(objects) == serialized + assert schema.loads(serialized) == objects From d610e5efc1e53bfb81aa8a3841cea9e2b5a6a7a8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 18:21:48 -0400 Subject: [PATCH 37/53] just use Any for now --- src/desert/_fields.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index d4a24d5..eaae1e0 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -166,7 +166,7 @@ class TaggedValue: class FromObjectProtocol(typing_extensions.Protocol): - def __call__(self, value: object) -> HintTagField: + def __call__(self, value: typing.Any) -> HintTagField: ... @@ -176,12 +176,12 @@ def __call__(self, tag: str) -> HintTagField: class FromTaggedProtocol(typing_extensions.Protocol): - def __call__(self, item: object) -> TaggedValue: + def __call__(self, item: typing.Any) -> TaggedValue: ... class ToTaggedProtocol(typing_extensions.Protocol): - def __call__(self, tag: object, value: object) -> object: + def __call__(self, tag: typing.Any, value: typing.Any) -> typing.Any: ... From 47197f6288efa8b9d62f77465f776ace577466a3 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Aug 2021 18:43:51 -0400 Subject: [PATCH 38/53] drop some args/kwargs to be more explicit --- src/desert/_fields.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index eaae1e0..813d5da 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -245,36 +245,43 @@ def to_externally_tagged(tag: str, value: typing.Any): return {tag: value} -@functools.wraps(TaggedUnion) -def externally_tagged_union(*args, **kwargs): +def externally_tagged_union( + from_object: FromObjectProtocol, + from_tag: FromTagProtocol, +): + # TODO: allow the pass through kwargs to the field + return TaggedUnion( - *args, + from_object=from_object, + from_tag=from_tag, from_tagged=from_externally_tagged, to_tagged=to_externally_tagged, - **kwargs, ) -def from_internally_tagged(item: typing.Any, type_key: str): +def from_internally_tagged(item: typing.Any, type_key: str) -> TaggedValue: return TaggedValue( tag=item[type_key], value={k: v for k, v in item.items() if k != type_key} ) -def to_internally_tagged(tag: str, value: typing.Any, type_key: str): +def to_internally_tagged(tag: str, value: typing.Any, type_key: str) -> typing.Any: if type_key in value: raise desert.exceptions.TypeKeyCollision(type_key=type_key, value=value) return {type_key: tag, **value} -@functools.wraps(TaggedUnion) -def internally_tagged_union(*args, type_key=default_tagged_type_key, **kwargs): +def internally_tagged_union( + from_object: FromObjectProtocol, + from_tag: FromTagProtocol, + type_key=default_tagged_type_key, +): return TaggedUnion( - *args, + from_object=from_object, + from_tag=from_tag, from_tagged=functools.partial(from_internally_tagged, type_key=type_key), to_tagged=functools.partial(to_internally_tagged, type_key=type_key), - **kwargs, ) @@ -292,20 +299,19 @@ def to_adjacently_tagged(tag: str, value: typing.Any, type_key: str, value_key: return {type_key: tag, value_key: value} -@functools.wraps(TaggedUnion) def adjacently_tagged_union( - *args, + from_object: FromObjectProtocol, + from_tag: FromTagProtocol, type_key=default_tagged_type_key, value_key=default_tagged_value_key, - **kwargs, ): return TaggedUnion( - *args, + from_object=from_object, + from_tag=from_tag, from_tagged=functools.partial( from_adjacently_tagged, type_key=type_key, value_key=value_key ), to_tagged=functools.partial( to_adjacently_tagged, type_key=type_key, value_key=value_key ), - **kwargs, ) From c2cf8a6d8571bbb8a15ea3446083767f14a79045 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 9 Aug 2021 11:51:33 -0400 Subject: [PATCH 39/53] docstrings and some more --- src/desert/_fields.py | 86 ++++++++++++++++++++++++++++++---------- src/desert/exceptions.py | 7 ++++ tests/test_fields.py | 22 ++++++---- 3 files changed, 87 insertions(+), 28 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 813d5da..1a75547 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -1,4 +1,3 @@ -import collections import functools import typing @@ -16,6 +15,13 @@ @attr.s(frozen=True, auto_attribs=True) class HintTagField: + """Serializing and deserializing a given piece of data requires a group of + information. A type hint that matches the data to be serialized, a Marshmallow + field that knows how to serialize and deserialize the data, and a string tag to + label the serialized data for deserialization. This is that group... There + must be a better name. + """ + hint: typing.Any tag: str field: marshmallow.fields.Field @@ -30,10 +36,12 @@ def register( ) -> None: ... - def from_object(self, value: typing.Any) -> HintTagField: + @property + def from_object(self) -> "FromObjectProtocol": ... - def from_tag(self, tag: str) -> HintTagField: + @property + def from_tag(self) -> "FromTagProtocol": ... @@ -76,8 +84,10 @@ def from_tag(self, tag: str) -> HintTagField: @check_field_registry_protocol @attr.s(auto_attribs=True) -class OrderedIsinstanceFieldRegistry: - the_list: typing.List[HintTagField] = attr.ib(factory=list) +class TypeAndHintFieldRegistry: + """This registry uses type hint and type checks to decide what field to use for + serialization. The deserialization field is chosen directly from the tag.""" + by_tag: typing.Dict[str, HintTagField] = attr.ib(factory=dict) # TODO: but type bans from-scratch metatypes... and protocols @@ -87,24 +97,18 @@ def register( tag: str, field: marshmallow.fields.Field, ) -> None: - # TODO: just disabling for now to show more interesting test results - # if any(key in self.the_dict for key in [cls, tag]): - # raise Exception() + if tag in self.by_tag: + raise desert.exceptions.TagAlreadyRegistered(tag=tag) type_tag_field = HintTagField(hint=hint, tag=tag, field=field) - self.the_list.append(type_tag_field) self.by_tag[tag] = type_tag_field - # # TODO: this type hinting... doesn't help much as it could return - # # another cls - # def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: - # return lambda cls: self.register(cls=cls, tag=tag, field=field) - def from_object(self, value: typing.Any) -> HintTagField: scores = {} - for type_tag_field in self.the_list: + # for type_tag_field in self.the_list: + for type_tag_field in self.by_tag.values(): score = 0 # if pytypes.is_of_type(value, type_tag_field.hint): @@ -141,7 +145,7 @@ def from_object(self, value: typing.Any) -> HintTagField: if high_score == 0: raise desert.exceptions.NoMatchingHintFound( - hints=[ttf.hint for ttf in self.the_list], value=value + hints=[ttf.hint for ttf in self.by_tag.values()], value=value ) potential = [ttf for ttf, score in scores.items() if score == high_score] @@ -185,7 +189,12 @@ def __call__(self, tag: typing.Any, value: typing.Any) -> typing.Any: ... -class TaggedUnion(marshmallow.fields.Field): +class TaggedUnionField(marshmallow.fields.Field): + """A Marshmallow field to handle unions where the data may not always be of a + single type. Usually this field would not be created directly but rather by + using helper functions to fill out the needed functions in a consistent manner. + """ + def __init__( self, *, @@ -235,13 +244,17 @@ def _serialize( default_tagged_value_key = "#value" -def from_externally_tagged(item: typing.Any): +def from_externally_tagged(item: typing.Any) -> TaggedValue: + """Process externally tagged data into a :class:`TaggedValue`.""" + [[tag, serialized_value]] = item.items() return TaggedValue(tag=tag, value=serialized_value) def to_externally_tagged(tag: str, value: typing.Any): + """Process untagged data to the externally tagged form.""" + return {tag: value} @@ -249,9 +262,11 @@ def externally_tagged_union( from_object: FromObjectProtocol, from_tag: FromTagProtocol, ): + """Create a :class:`TaggedUnionField` that supports the externally tagged form.""" + # TODO: allow the pass through kwargs to the field - return TaggedUnion( + return TaggedUnionField( from_object=from_object, from_tag=from_tag, from_tagged=from_externally_tagged, @@ -260,12 +275,16 @@ def externally_tagged_union( def from_internally_tagged(item: typing.Any, type_key: str) -> TaggedValue: + """Process internally tagged data into a :class:`TaggedValue`.""" + return TaggedValue( tag=item[type_key], value={k: v for k, v in item.items() if k != type_key} ) def to_internally_tagged(tag: str, value: typing.Any, type_key: str) -> typing.Any: + """Process untagged data to the internally tagged form.""" + if type_key in value: raise desert.exceptions.TypeKeyCollision(type_key=type_key, value=value) @@ -277,7 +296,9 @@ def internally_tagged_union( from_tag: FromTagProtocol, type_key=default_tagged_type_key, ): - return TaggedUnion( + """Create a :class:`TaggedUnionField` that supports the internally tagged form.""" + + return TaggedUnionField( from_object=from_object, from_tag=from_tag, from_tagged=functools.partial(from_internally_tagged, type_key=type_key), @@ -286,6 +307,8 @@ def internally_tagged_union( def from_adjacently_tagged(item: typing.Any, type_key: str, value_key: str): + """Process adjacently tagged data into a :class:`TaggedValue`.""" + tag = item.pop(type_key) serialized_value = item.pop(value_key) @@ -296,6 +319,8 @@ def from_adjacently_tagged(item: typing.Any, type_key: str, value_key: str): def to_adjacently_tagged(tag: str, value: typing.Any, type_key: str, value_key: str): + """Process untagged data to the adjacently tagged form.""" + return {type_key: tag, value_key: value} @@ -305,7 +330,9 @@ def adjacently_tagged_union( type_key=default_tagged_type_key, value_key=default_tagged_value_key, ): - return TaggedUnion( + """Create a :class:`TaggedUnionField` that supports the adjacently tagged form.""" + + return TaggedUnionField( from_object=from_object, from_tag=from_tag, from_tagged=functools.partial( @@ -315,3 +342,20 @@ def adjacently_tagged_union( to_adjacently_tagged, type_key=type_key, value_key=value_key ), ) + + +def adjacently_tagged_union_from_registry( + registry: FieldRegistry, + type_key=default_tagged_type_key, + value_key=default_tagged_value_key, +) -> TaggedUnionField: + """Create a :class:`TaggedUnionField` that supports the adjacently tagged form + from a :class:`FieldRegistry`. + """ + + return adjacently_tagged_union( + from_object=registry.from_object, + from_tag=registry.from_tag, + type_key=type_key, + value_key=value_key, + ) diff --git a/src/desert/exceptions.py b/src/desert/exceptions.py index 8bbb47c..6682c9e 100644 --- a/src/desert/exceptions.py +++ b/src/desert/exceptions.py @@ -36,6 +36,13 @@ class NotAnAttrsClassOrDataclass(DesertException): """Raised for dataclass operations on non-dataclasses.""" +class TagAlreadyRegistered(DesertException): + """Raised when registering a tag that has already been registered.""" + + def __init__(self, tag: str): + super().__init__(f"Tag already registered: {tag!r}") + + class TypeKeyCollision(DesertException): """Raised when a tag key collides with a data value.""" diff --git a/tests/test_fields.py b/tests/test_fields.py index 9e80e1e..6edb907 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -90,7 +90,7 @@ def build( to_serialize=("def", "13"), serialized=["def", "13"], deserialized=["def", "13"], - tag="string_list_tag", + tag="string_sequence_tag", field=marshmallow.fields.List(marshmallow.fields.String()), ), ] @@ -156,7 +156,7 @@ def _custom_example_data(request): def build_order_isinstance_registry(examples): - registry = desert._fields.OrderedIsinstanceFieldRegistry() + registry = desert._fields.TypeAndHintFieldRegistry() # registry.register( # hint=typing.List[], @@ -201,7 +201,7 @@ class C: def test_registry_raises_for_multiple_matches(): - registry = desert._fields.OrderedIsinstanceFieldRegistry() + registry = desert._fields.TypeAndHintFieldRegistry() registry.register( hint=typing.Sequence, @@ -337,13 +337,11 @@ def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): def test_actual_example(): - registry = desert._fields.OrderedIsinstanceFieldRegistry() + registry = desert._fields.TypeAndHintFieldRegistry() registry.register(hint=str, tag="str", field=marshmallow.fields.String()) registry.register(hint=int, tag="int", field=marshmallow.fields.Integer()) - field = desert._fields.adjacently_tagged_union( - from_object=registry.from_object, from_tag=registry.from_tag - ) + field = desert._fields.adjacently_tagged_union_from_registry(registry=registry) @attr.frozen class C: @@ -358,3 +356,13 @@ class C: assert schema.dumps(objects) == serialized assert schema.loads(serialized) == objects + + +def test_raises_for_tag_reregistration(): + registry = desert._fields.TypeAndHintFieldRegistry() + registry.register(hint=str, tag="duplicate_tag", field=marshmallow.fields.String()) + + with pytest.raises(desert.exceptions.TagAlreadyRegistered): + registry.register( + hint=int, tag="duplicate_tag", field=marshmallow.fields.Integer() + ) From 6722760a5760e26e37d46090c75e084901c9cf37 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 17 Aug 2021 13:10:43 -0400 Subject: [PATCH 40/53] use typing_inspect.get_origin() instead of .__origin__ --- src/desert/_fields.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 1a75547..0d053ca 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -5,6 +5,7 @@ import marshmallow.fields import typeguard import typing_extensions +import typing_inspect import desert._util import desert.exceptions @@ -133,11 +134,8 @@ def from_object(self, value: typing.Any) -> HintTagField: # Only use this to disambiguate between already selected options such # as ["a", "b"] matching both typing.List[str] and typing.Sequence[str]. # This only works properly on 3.7+. - try: - if type(value) == type_tag_field.hint.__origin__: - score += 1 - except (AttributeError, TypeError): - pass + if type(value) == typing_inspect.get_origin(type_tag_field.hint): + score += 1 scores[type_tag_field] = score From 76bc4da30efc6f23dd1764927a15cd0a798b00ee Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 19 Aug 2021 10:00:11 -0400 Subject: [PATCH 41/53] type hinting catchup --- src/desert/__init__.py | 2 +- src/desert/_fields.py | 106 ++++++++++++++++++++--------------- tests/test_fields.py | 123 ++++++++++++++++++++++++++--------------- tests/test_make.py | 6 +- 4 files changed, 144 insertions(+), 93 deletions(-) diff --git a/src/desert/__init__.py b/src/desert/__init__.py index c55d5e8..eb26321 100644 --- a/src/desert/__init__.py +++ b/src/desert/__init__.py @@ -153,7 +153,7 @@ def ib( *, metadata: t.Mapping[object, object] = {}, **kw: object, -) -> object: +) -> t.Any: ... diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 0d053ca..b677e88 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -1,5 +1,5 @@ import functools -import typing +import typing as t import attr import marshmallow.fields @@ -11,7 +11,7 @@ import desert.exceptions -T = typing.TypeVar("T") +T = t.TypeVar("T") @attr.s(frozen=True, auto_attribs=True) @@ -23,7 +23,7 @@ class HintTagField: must be a better name. """ - hint: typing.Any + hint: t.Any tag: str field: marshmallow.fields.Field @@ -31,7 +31,7 @@ class HintTagField: class FieldRegistry(typing_extensions.Protocol): def register( self, - hint: typing.Any, + hint: t.Any, tag: str, field: marshmallow.fields.Field, ) -> None: @@ -51,14 +51,14 @@ def from_tag(self) -> "FromTagProtocol": # @attr.s(auto_attribs=True) # class TypeDictFieldRegistry: -# the_dict: typing.Dict[ -# typing.Union[type, str], +# the_dict: t.Dict[ +# t.Union[type, str], # HintTagField, # ] = attr.ib(factory=dict) # # def register( # self, -# hint: typing.Any, +# hint: t.Any, # tag: str, # field: marshmallow.fields.Field, # ) -> None: @@ -73,10 +73,10 @@ def from_tag(self) -> "FromTagProtocol": # # # # TODO: this type hinting... doesn't help much as it could return # # # another cls -# # def __call__(self, tag: str, field: marshmallow.fields) -> typing.Callable[[T], T]: +# # def __call__(self, tag: str, field: marshmallow.fields) -> t.Callable[[T], T]: # # return lambda cls: self.register(cls=cls, tag=tag, field=field) # -# def from_object(self, value: typing.Any) -> HintTagField: +# def from_object(self, value: t.Any) -> HintTagField: # return self.the_dict[type(value)] # # def from_tag(self, tag: str) -> HintTagField: @@ -89,12 +89,12 @@ class TypeAndHintFieldRegistry: """This registry uses type hint and type checks to decide what field to use for serialization. The deserialization field is chosen directly from the tag.""" - by_tag: typing.Dict[str, HintTagField] = attr.ib(factory=dict) + by_tag: t.Dict[str, HintTagField] = attr.ib(factory=dict) # TODO: but type bans from-scratch metatypes... and protocols def register( self, - hint: typing.Any, + hint: t.Any, tag: str, field: marshmallow.fields.Field, ) -> None: @@ -105,7 +105,7 @@ def register( self.by_tag[tag] = type_tag_field - def from_object(self, value: typing.Any) -> HintTagField: + def from_object(self, value: object) -> HintTagField: scores = {} # for type_tag_field in self.the_list: @@ -132,7 +132,7 @@ def from_object(self, value: typing.Any) -> HintTagField: if score > 0: # Only use this to disambiguate between already selected options such - # as ["a", "b"] matching both typing.List[str] and typing.Sequence[str]. + # as ["a", "b"] matching both t.List[str] and t.Sequence[str]. # This only works properly on 3.7+. if type(value) == typing_inspect.get_origin(type_tag_field.hint): score += 1 @@ -164,11 +164,11 @@ def from_tag(self, tag: str) -> HintTagField: @attr.s(auto_attribs=True) class TaggedValue: tag: str - value: typing.Any + value: object class FromObjectProtocol(typing_extensions.Protocol): - def __call__(self, value: typing.Any) -> HintTagField: + def __call__(self, value: object) -> HintTagField: ... @@ -178,12 +178,12 @@ def __call__(self, tag: str) -> HintTagField: class FromTaggedProtocol(typing_extensions.Protocol): - def __call__(self, item: typing.Any) -> TaggedValue: + def __call__(self, item: t.Any) -> TaggedValue: ... class ToTaggedProtocol(typing_extensions.Protocol): - def __call__(self, tag: typing.Any, value: typing.Any) -> typing.Any: + def __call__(self, tag: str, value: t.Any) -> object: ... @@ -200,8 +200,10 @@ def __init__( from_tag: FromTagProtocol, from_tagged: FromTaggedProtocol, to_tagged: ToTaggedProtocol, - **kwargs, - ): + # object results in the super() call complaining about types + # https://github.com/python/mypy/issues/5382 + **kwargs: t.Any, + ) -> None: super().__init__(**kwargs) self.from_object = from_object @@ -211,11 +213,13 @@ def __init__( def _deserialize( self, - value: typing.Any, - attr: typing.Optional[str], - data: typing.Optional[typing.Mapping[str, typing.Any]], - **kwargs, - ) -> typing.Any: + value: object, + attr: t.Optional[str], + data: t.Optional[t.Mapping[str, object]], + # object results in the super() call complaining about types + # https://github.com/python/mypy/issues/5382 + **kwargs: t.Any, + ) -> object: tagged_value = self.from_tagged(item=value) type_tag_field = self.from_tag(tagged_value.tag) @@ -225,11 +229,13 @@ def _deserialize( def _serialize( self, - value: typing.Any, + value: object, attr: str, - obj: typing.Any, - **kwargs, - ) -> typing.Any: + obj: object, + # object results in the super() call complaining about types + # https://github.com/python/mypy/issues/5382 + **kwargs: t.Any, + ) -> object: type_tag_field = self.from_object(value) field = type_tag_field.field tag = type_tag_field.tag @@ -242,7 +248,7 @@ def _serialize( default_tagged_value_key = "#value" -def from_externally_tagged(item: typing.Any) -> TaggedValue: +def from_externally_tagged(item: t.Mapping[str, object]) -> TaggedValue: """Process externally tagged data into a :class:`TaggedValue`.""" [[tag, serialized_value]] = item.items() @@ -250,7 +256,7 @@ def from_externally_tagged(item: typing.Any) -> TaggedValue: return TaggedValue(tag=tag, value=serialized_value) -def to_externally_tagged(tag: str, value: typing.Any): +def to_externally_tagged(tag: str, value: object) -> t.Dict[str, object]: """Process untagged data to the externally tagged form.""" return {tag: value} @@ -259,7 +265,7 @@ def to_externally_tagged(tag: str, value: typing.Any): def externally_tagged_union( from_object: FromObjectProtocol, from_tag: FromTagProtocol, -): +) -> TaggedUnionField: """Create a :class:`TaggedUnionField` that supports the externally tagged form.""" # TODO: allow the pass through kwargs to the field @@ -272,15 +278,23 @@ def externally_tagged_union( ) -def from_internally_tagged(item: typing.Any, type_key: str) -> TaggedValue: +def from_internally_tagged(item: t.Mapping[str, object], type_key: str) -> TaggedValue: """Process internally tagged data into a :class:`TaggedValue`.""" + # it just kind of has to be a string... + type_string: str = item[type_key] # type: ignore[assignment] + return TaggedValue( - tag=item[type_key], value={k: v for k, v in item.items() if k != type_key} + tag=type_string, + value={k: v for k, v in item.items() if k != type_key}, ) -def to_internally_tagged(tag: str, value: typing.Any, type_key: str) -> typing.Any: +def to_internally_tagged( + tag: str, + value: t.Mapping[str, object], + type_key: str, +) -> t.Mapping[str, object]: """Process untagged data to the internally tagged form.""" if type_key in value: @@ -292,8 +306,8 @@ def to_internally_tagged(tag: str, value: typing.Any, type_key: str) -> typing.A def internally_tagged_union( from_object: FromObjectProtocol, from_tag: FromTagProtocol, - type_key=default_tagged_type_key, -): + type_key: str = default_tagged_type_key, +) -> TaggedUnionField: """Create a :class:`TaggedUnionField` that supports the internally tagged form.""" return TaggedUnionField( @@ -304,10 +318,12 @@ def internally_tagged_union( ) -def from_adjacently_tagged(item: typing.Any, type_key: str, value_key: str): +def from_adjacently_tagged( + item: t.Dict[str, object], type_key: str, value_key: str +) -> TaggedValue: """Process adjacently tagged data into a :class:`TaggedValue`.""" - tag = item.pop(type_key) + tag: str = item.pop(type_key) # type: ignore[assignment] serialized_value = item.pop(value_key) if len(item) > 0: @@ -316,7 +332,9 @@ def from_adjacently_tagged(item: typing.Any, type_key: str, value_key: str): return TaggedValue(tag=tag, value=serialized_value) -def to_adjacently_tagged(tag: str, value: typing.Any, type_key: str, value_key: str): +def to_adjacently_tagged( + tag: str, value: object, type_key: str, value_key: str +) -> t.Dict[str, object]: """Process untagged data to the adjacently tagged form.""" return {type_key: tag, value_key: value} @@ -325,9 +343,9 @@ def to_adjacently_tagged(tag: str, value: typing.Any, type_key: str, value_key: def adjacently_tagged_union( from_object: FromObjectProtocol, from_tag: FromTagProtocol, - type_key=default_tagged_type_key, - value_key=default_tagged_value_key, -): + type_key: str = default_tagged_type_key, + value_key: str = default_tagged_value_key, +) -> TaggedUnionField: """Create a :class:`TaggedUnionField` that supports the adjacently tagged form.""" return TaggedUnionField( @@ -344,8 +362,8 @@ def adjacently_tagged_union( def adjacently_tagged_union_from_registry( registry: FieldRegistry, - type_key=default_tagged_type_key, - value_key=default_tagged_value_key, + type_key: str = default_tagged_type_key, + value_key: str = default_tagged_value_key, ) -> TaggedUnionField: """Create a :class:`TaggedUnionField` that supports the adjacently tagged form from a :class:`FieldRegistry`. diff --git a/tests/test_fields.py b/tests/test_fields.py index 6edb907..05fd1d1 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2,8 +2,10 @@ import collections.abc import decimal import json -import typing +import typing as t +# https://github.com/pytest-dev/pytest/issues/7469 +import _pytest.fixtures import attr import marshmallow import pytest @@ -19,24 +21,24 @@ @attr.frozen class ExampleData: - to_serialize: typing.Any - serialized: typing.Any - deserialized: typing.Any + to_serialize: t.Any + serialized: t.Any + deserialized: t.Any tag: str field: marshmallow.fields.Field # TODO: can we be more specific? - hint: typing.Any + hint: t.Any @classmethod def build( cls, - hint, - to_serialize, - tag, - field, - serialized=_NOTHING, - deserialized=_NOTHING, - ): + hint: object, + to_serialize: object, + tag: str, + field: marshmallow.fields.Field, + serialized: object = _NOTHING, + deserialized: object = _NOTHING, + ) -> "ExampleData": if serialized is _NOTHING: serialized = to_serialize @@ -74,19 +76,19 @@ def build( field=marshmallow.fields.Decimal(as_string=True), ), ExampleData.build( - hint=typing.List[int], + hint=t.List[int], to_serialize=[1, 2, 3], tag="integer_list_tag", field=marshmallow.fields.List(marshmallow.fields.Integer()), ), ExampleData.build( - hint=typing.List[str], + hint=t.List[str], to_serialize=["abc", "2", "mno"], tag="string_list_tag", field=marshmallow.fields.List(marshmallow.fields.String()), ), ExampleData.build( - hint=typing.Sequence[str], + hint=t.Sequence[str], to_serialize=("def", "13"), serialized=["def", "13"], deserialized=["def", "13"], @@ -121,8 +123,8 @@ class CustomExampleClass: params=all_example_data_list, ids=[str(example) for example in all_example_data_list], ) -def _example_data(request): - return request.param +def _example_data(request: _pytest.fixtures.SubRequest) -> ExampleData: + return request.param # type: ignore[no-any-return] @pytest.fixture( @@ -130,8 +132,8 @@ def _example_data(request): params=custom_example_data_list, ids=[str(example) for example in custom_example_data_list], ) -def _custom_example_data(request): - return request.param +def _custom_example_data(request: _pytest.fixtures.SubRequest) -> ExampleData: + return request.param # type: ignore[no-any-return] # def build_type_dict_registry(examples): @@ -155,11 +157,13 @@ def _custom_example_data(request): # ) -def build_order_isinstance_registry(examples): +def build_order_isinstance_registry( + examples: t.List[ExampleData], +) -> desert._fields.TypeAndHintFieldRegistry: registry = desert._fields.TypeAndHintFieldRegistry() # registry.register( - # hint=typing.List[], + # hint=t.List[], # tag="sequence_abc", # field=marshmallow.fields.List(marshmallow.fields.String()), # ) @@ -186,11 +190,15 @@ def build_order_isinstance_registry(examples): params=registries, ids=registry_ids, ) -def _registry(request): - return request.param +def _registry( + request: _pytest.fixtures.SubRequest, +) -> desert._fields.TypeAndHintFieldRegistry: + return request.param # type: ignore[no-any-return] -def test_registry_raises_for_no_match(registry): +def test_registry_raises_for_no_match( + registry: desert._fields.FieldRegistry, +) -> None: class C: pass @@ -200,17 +208,17 @@ class C: registry.from_object(value=c) -def test_registry_raises_for_multiple_matches(): +def test_registry_raises_for_multiple_matches() -> None: registry = desert._fields.TypeAndHintFieldRegistry() registry.register( - hint=typing.Sequence, + hint=t.Sequence, tag="sequence", field=marshmallow.fields.List(marshmallow.fields.Field()), ) registry.register( - hint=typing.Collection, + hint=t.Collection, tag="collection", field=marshmallow.fields.List(marshmallow.fields.Field()), ) @@ -220,14 +228,18 @@ def test_registry_raises_for_multiple_matches(): @pytest.fixture(name="externally_tagged_field") -def _externally_tagged_field(registry): +def _externally_tagged_field( + registry: desert._fields.FieldRegistry, +) -> desert._fields.TaggedUnionField: return desert._fields.externally_tagged_union( from_object=registry.from_object, from_tag=registry.from_tag, ) -def test_externally_tagged_deserialize(example_data, externally_tagged_field): +def test_externally_tagged_deserialize( + example_data: ExampleData, externally_tagged_field: desert._fields.TaggedUnionField +) -> None: serialized = {example_data.tag: example_data.serialized} deserialized = externally_tagged_field.deserialize(serialized) @@ -238,9 +250,9 @@ def test_externally_tagged_deserialize(example_data, externally_tagged_field): def test_externally_tagged_deserialize_extra_key_raises( - example_data, - externally_tagged_field, -): + example_data: ExampleData, + externally_tagged_field: desert._fields.TaggedUnionField, +) -> None: serialized = { example_data.tag: { "#value": example_data.serialized, @@ -252,7 +264,10 @@ def test_externally_tagged_deserialize_extra_key_raises( externally_tagged_field.deserialize(serialized) -def test_externally_tagged_serialize(example_data, externally_tagged_field): +def test_externally_tagged_serialize( + example_data: ExampleData, + externally_tagged_field: desert._fields.TaggedUnionField, +) -> None: obj = {"key": example_data.to_serialize} serialized = externally_tagged_field.serialize("key", obj) @@ -261,21 +276,26 @@ def test_externally_tagged_serialize(example_data, externally_tagged_field): @pytest.fixture(name="internally_tagged_field") -def _internally_tagged_field(registry): +def _internally_tagged_field( + registry: desert._fields.FieldRegistry, +) -> desert._fields.TaggedUnionField: return desert._fields.internally_tagged_union( from_object=registry.from_object, from_tag=registry.from_tag, ) -def test_to_internally_tagged_raises_for_tag_collision(): +def test_to_internally_tagged_raises_for_tag_collision() -> None: with pytest.raises(desert.exceptions.TypeKeyCollision): desert._fields.to_internally_tagged( tag="C", value={"collide": True}, type_key="collide" ) -def test_internally_tagged_deserialize(custom_example_data, internally_tagged_field): +def test_internally_tagged_deserialize( + custom_example_data: ExampleData, + internally_tagged_field: desert._fields.TaggedUnionField, +) -> None: serialized = {"#type": custom_example_data.tag, **custom_example_data.serialized} deserialized = internally_tagged_field.deserialize(serialized) @@ -285,7 +305,10 @@ def test_internally_tagged_deserialize(custom_example_data, internally_tagged_fi assert (type(deserialized) == type(expected)) and (deserialized == expected) -def test_internally_tagged_serialize(custom_example_data, internally_tagged_field): +def test_internally_tagged_serialize( + custom_example_data: ExampleData, + internally_tagged_field: desert._fields.TaggedUnionField, +) -> None: obj = {"key": custom_example_data.to_serialize} serialized = internally_tagged_field.serialize("key", obj) @@ -297,14 +320,19 @@ def test_internally_tagged_serialize(custom_example_data, internally_tagged_fiel @pytest.fixture(name="adjacently_tagged_field") -def _adjacently_tagged_field(registry): +def _adjacently_tagged_field( + registry: desert._fields.FieldRegistry, +) -> desert._fields.TaggedUnionField: return desert._fields.adjacently_tagged_union( from_object=registry.from_object, from_tag=registry.from_tag, ) -def test_adjacently_tagged_deserialize(example_data, adjacently_tagged_field): +def test_adjacently_tagged_deserialize( + example_data: ExampleData, + adjacently_tagged_field: desert._fields.TaggedUnionField, +) -> None: serialized = {"#type": example_data.tag, "#value": example_data.serialized} deserialized = adjacently_tagged_field.deserialize(serialized) @@ -315,9 +343,9 @@ def test_adjacently_tagged_deserialize(example_data, adjacently_tagged_field): def test_adjacently_tagged_deserialize_extra_key_raises( - example_data, - adjacently_tagged_field, -): + example_data: ExampleData, + adjacently_tagged_field: desert._fields.TaggedUnionField, +) -> None: serialized = { "#type": example_data.tag, "#value": example_data.serialized, @@ -328,7 +356,10 @@ def test_adjacently_tagged_deserialize_extra_key_raises( adjacently_tagged_field.deserialize(serialized) -def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): +def test_adjacently_tagged_serialize( + example_data: ExampleData, + adjacently_tagged_field: desert._fields.TaggedUnionField, +) -> None: obj = {"key": example_data.to_serialize} serialized = adjacently_tagged_field.serialize("key", obj) @@ -336,7 +367,7 @@ def test_adjacently_tagged_serialize(example_data, adjacently_tagged_field): assert serialized == {"#type": example_data.tag, "#value": example_data.serialized} -def test_actual_example(): +def test_actual_example() -> None: registry = desert._fields.TypeAndHintFieldRegistry() registry.register(hint=str, tag="str", field=marshmallow.fields.String()) registry.register(hint=int, tag="int", field=marshmallow.fields.Integer()) @@ -346,7 +377,7 @@ def test_actual_example(): @attr.frozen class C: # TODO: desert.ib() shouldn't be needed for many cases - union: typing.Union[str, int] = desert.ib(marshmallow_field=field) + union: t.Union[str, int] = desert.ib(marshmallow_field=field) schema = desert.schema(C) @@ -358,7 +389,7 @@ class C: assert schema.loads(serialized) == objects -def test_raises_for_tag_reregistration(): +def test_raises_for_tag_reregistration() -> None: registry = desert._fields.TypeAndHintFieldRegistry() registry.register(hint=str, tag="duplicate_tag", field=marshmallow.fields.String()) diff --git a/tests/test_make.py b/tests/test_make.py index 8efdcdb..fdd514e 100644 --- a/tests/test_make.py +++ b/tests/test_make.py @@ -259,7 +259,7 @@ def test_concise_attrib() -> None: @attr.dataclass class A: - x: datetime.datetime = desert.ib(marshmallow.fields.NaiveDateTime()) # type: ignore[assignment] + x: datetime.datetime = desert.ib(marshmallow.fields.NaiveDateTime()) timestring = "2019-10-21T10:25:00" dt = datetime.datetime(year=2019, month=10, day=21, hour=10, minute=25, second=00) @@ -288,7 +288,9 @@ def test_concise_attrib_metadata() -> None: @attr.dataclass class A: - x: datetime.datetime = desert.ib(marshmallow.fields.NaiveDateTime(), metadata={"foo": 1}) # type: ignore[assignment] + x: datetime.datetime = desert.ib( + marshmallow.fields.NaiveDateTime(), metadata={"foo": 1} + ) timestring = "2019-10-21T10:25:00" dt = datetime.datetime(year=2019, month=10, day=21, hour=10, minute=25, second=00) From aa852309bd3e241c61df526f92645db2a0a1590d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 23 Aug 2021 17:43:34 -0400 Subject: [PATCH 42/53] tagged union documentation --- docs/reference/fields.rst | 103 ++++++++++++++++++++++++++++++++++++++ docs/reference/index.rst | 1 + src/desert/_fields.py | 79 ++++++++++++++++++++++++++--- tests/test_fields.py | 10 ++-- 4 files changed, 182 insertions(+), 11 deletions(-) create mode 100644 docs/reference/fields.rst diff --git a/docs/reference/fields.rst b/docs/reference/fields.rst new file mode 100644 index 0000000..c08307b --- /dev/null +++ b/docs/reference/fields.rst @@ -0,0 +1,103 @@ +.. + TODO: figure out proper location for this stuff including making it public + if in desert + + +desert._fields module +===================== + +Tagged Unions +------------- + +Serializing and deserializing data of uncertain type can be tricky. +Cases where the type is hinted with :class:`typing.Union` create such cases. +Some solutions have a list of possible types and use the first one that works. +This can be used in some cases where the data for different types is sufficiently unique as to only work with a single type. +In more general cases you have difficulties where data of one type ends up getting processed by another type that is similar, but not the same. + +In an effort to reduce the heuristics involved in serialization and deserialization an explicit tag can be added to identify the type. +That is the basic feature of this group of utilities related to tagged unions. + +A tag indicating the object's type can be applied in various ways. +Presently three forms are implemented: adjacently tagged, internally tagged, and externally tagged. +Adjacently tagged is the most explicit form and is the recommended default. +You can write your own helper functions to implement your own tagging form if needed and still make use of the rest of the mechanisms implemented here. + +- A class definition and bare serialized object for reference + + .. code-block:: python + + @dataclasses.dataclass + class Cat: + name: str + color: str + + .. code-block:: json + + { + "name": "Max", + "color": "tuxedo", + } + +- Adjacently tagged + + .. include:: ../snippets/tag_forms/adjacent.rst + +- Internally tagged + + .. include:: ../snippets/tag_forms/internal.rst + +- Externally tagged + + .. include:: ../snippets/tag_forms/external.rst + +The code below is an actual test from the Desert test suite that provides an example usage of the tools that will be covered in detail below. + +.. literalinclude:: ../../tests/test_fields.py + :start-after: # start tagged_union_example + :end-before: # end tagged_union_example + + +Fields +...... + +A :class:`marshmallow.fields.Field` is needed to describe the serialization. +This role is filled by :class:`desert._fields.TaggedUnionField`. +Several helpers at different levels are included to generate field instances that support each of the tagging schemes shown above. +:ref:`Registries ` are used to collect and hold the information needed to make the choices the field needs. +The helpers below create :class:`desert._fields.TaggedUnionField` instances that are backed by the passed registry. + +.. autofunction:: desert._fields.adjacently_tagged_union_from_registry +.. autofunction:: desert._fields.internally_tagged_union_from_registry +.. autofunction:: desert._fields.externally_tagged_union_from_registry + +.. autoclass:: desert._fields.TaggedUnionField + :members: + :undoc-members: + :show-inheritance: + + +.. _tagged_union_registries: + +Registries +.......... + +Since unions are inherently about handling multiple types, fields that handle unions must be able to make decisions about multiple types. +Registries are not required to leverage other pieces of union support if you are developing their logic yourself. +If you are using the builtin mechanisms then a registry will be needed to define the relationships between tags, fields, and object types. + +.. + TODO: sure seems like the user shouldn't need to call Nested() themselves + +The registry's :meth:`desert._fields.FieldRegistryProtocol.register` method will primarily be used. +As an example, you might register a custom class ``Cat`` by providing a hint of ``Cat``, a tag of ``"cat"``, and a field such as ``marshmallow.fields.Nested(desert.schema(Cat))``. + +.. autoclass:: desert._fields.FieldRegistryProtocol + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: desert._fields.TypeAndHintFieldRegistry + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/reference/index.rst b/docs/reference/index.rst index a981732..9f266bc 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -5,3 +5,4 @@ Reference :glob: desert* + fields diff --git a/src/desert/_fields.py b/src/desert/_fields.py index b677e88..93f95e2 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -14,6 +14,7 @@ T = t.TypeVar("T") +# TODO: there must be a better name @attr.s(frozen=True, auto_attribs=True) class HintTagField: """Serializing and deserializing a given piece of data requires a group of @@ -28,25 +29,39 @@ class HintTagField: field: marshmallow.fields.Field -class FieldRegistry(typing_extensions.Protocol): +class FieldRegistryProtocol(typing_extensions.Protocol): + """This protocol encourages registries to provide a common interface. The actual + implementation of the mapping from objects to be serialized to their Marshmallow + fields, and likewise from the serialized data, can take any form. + """ + def register( self, hint: t.Any, tag: str, field: marshmallow.fields.Field, ) -> None: + """Inform the registry of the relationship between the passed hint, tag, and + field. + """ ... @property def from_object(self) -> "FromObjectProtocol": + """This is a funny way of writing that the registry's `.from_object()` method + should satisfy :class:`FromObjectProtocol`. + """ ... @property def from_tag(self) -> "FromTagProtocol": + """This is a funny way of writing that the registry's `.from_tag()` method + should satisfy :class:`FromTagProtocol`. + """ ... -check_field_registry_protocol = desert._util.ProtocolChecker[FieldRegistry]() +check_field_registry_protocol = desert._util.ProtocolChecker[FieldRegistryProtocol]() # @attr.s(auto_attribs=True) @@ -86,8 +101,9 @@ def from_tag(self) -> "FromTagProtocol": @check_field_registry_protocol @attr.s(auto_attribs=True) class TypeAndHintFieldRegistry: - """This registry uses type hint and type checks to decide what field to use for - serialization. The deserialization field is chosen directly from the tag.""" + """This registry uses type and type hint checks to decide what field to use for + serialization. The deserialization field is chosen directly from the tag. + """ by_tag: t.Dict[str, HintTagField] = attr.ib(factory=dict) @@ -191,6 +207,21 @@ class TaggedUnionField(marshmallow.fields.Field): """A Marshmallow field to handle unions where the data may not always be of a single type. Usually this field would not be created directly but rather by using helper functions to fill out the needed functions in a consistent manner. + + Helpers are provided both to directly create various forms of this field as well + as to create the same from a :class:`FieldRegistry`. + + - From a registry + + - :func:`externally_tagged_union_from_registry` + - :func:`internally_tagged_union_from_registry` + - :func:`adjacently_tagged_union_from_registry` + + - Direct + + - :func:`externally_tagged_union` + - :func:`internally_tagged_union` + - :func:`adjacently_tagged_union` """ def __init__( @@ -278,6 +309,21 @@ def externally_tagged_union( ) +def externally_tagged_union_from_registry( + registry: FieldRegistryProtocol, +) -> TaggedUnionField: + """Use a :class:`FieldRegistry` to create a :class:`TaggedUnionField` that supports + the externally tagged form. Externally tagged data has the following form. + + .. include:: ../snippets/tag_forms/external.rst + """ + + return externally_tagged_union( + from_object=registry.from_object, + from_tag=registry.from_tag, + ) + + def from_internally_tagged(item: t.Mapping[str, object], type_key: str) -> TaggedValue: """Process internally tagged data into a :class:`TaggedValue`.""" @@ -318,6 +364,23 @@ def internally_tagged_union( ) +def internally_tagged_union_from_registry( + registry: FieldRegistryProtocol, + type_key: str = default_tagged_type_key, +) -> TaggedUnionField: + """Use a :class:`FieldRegistry` to create a :class:`TaggedUnionField` that supports + the internally tagged form. Internally tagged data has the following form. + + .. include:: ../snippets/tag_forms/internal.rst + """ + + return internally_tagged_union( + from_object=registry.from_object, + from_tag=registry.from_tag, + type_key=type_key, + ) + + def from_adjacently_tagged( item: t.Dict[str, object], type_key: str, value_key: str ) -> TaggedValue: @@ -361,12 +424,14 @@ def adjacently_tagged_union( def adjacently_tagged_union_from_registry( - registry: FieldRegistry, + registry: FieldRegistryProtocol, type_key: str = default_tagged_type_key, value_key: str = default_tagged_value_key, ) -> TaggedUnionField: - """Create a :class:`TaggedUnionField` that supports the adjacently tagged form - from a :class:`FieldRegistry`. + """Use a :class:`FieldRegistry` to create a :class:`TaggedUnionField` that supports + the adjacently tagged form. Adjacently tagged data has the following form. + + .. include:: ../snippets/tag_forms/adjacent.rst """ return adjacently_tagged_union( diff --git a/tests/test_fields.py b/tests/test_fields.py index 05fd1d1..e60e53a 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -197,7 +197,7 @@ def _registry( def test_registry_raises_for_no_match( - registry: desert._fields.FieldRegistry, + registry: desert._fields.FieldRegistryProtocol, ) -> None: class C: pass @@ -229,7 +229,7 @@ def test_registry_raises_for_multiple_matches() -> None: @pytest.fixture(name="externally_tagged_field") def _externally_tagged_field( - registry: desert._fields.FieldRegistry, + registry: desert._fields.FieldRegistryProtocol, ) -> desert._fields.TaggedUnionField: return desert._fields.externally_tagged_union( from_object=registry.from_object, @@ -277,7 +277,7 @@ def test_externally_tagged_serialize( @pytest.fixture(name="internally_tagged_field") def _internally_tagged_field( - registry: desert._fields.FieldRegistry, + registry: desert._fields.FieldRegistryProtocol, ) -> desert._fields.TaggedUnionField: return desert._fields.internally_tagged_union( from_object=registry.from_object, @@ -321,7 +321,7 @@ def test_internally_tagged_serialize( @pytest.fixture(name="adjacently_tagged_field") def _adjacently_tagged_field( - registry: desert._fields.FieldRegistry, + registry: desert._fields.FieldRegistryProtocol, ) -> desert._fields.TaggedUnionField: return desert._fields.adjacently_tagged_union( from_object=registry.from_object, @@ -367,6 +367,7 @@ def test_adjacently_tagged_serialize( assert serialized == {"#type": example_data.tag, "#value": example_data.serialized} +# start tagged_union_example def test_actual_example() -> None: registry = desert._fields.TypeAndHintFieldRegistry() registry.register(hint=str, tag="str", field=marshmallow.fields.String()) @@ -387,6 +388,7 @@ class C: assert schema.dumps(objects) == serialized assert schema.loads(serialized) == objects +# end tagged_union_example def test_raises_for_tag_reregistration() -> None: From 64a20d92847ccff9007614ea93fc20976fad8bb7 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 23 Aug 2021 17:50:45 -0400 Subject: [PATCH 43/53] add missing snippets --- docs/snippets/tag_forms/adjacent.rst | 9 +++++++++ docs/snippets/tag_forms/external.rst | 8 ++++++++ docs/snippets/tag_forms/internal.rst | 7 +++++++ 3 files changed, 24 insertions(+) create mode 100644 docs/snippets/tag_forms/adjacent.rst create mode 100644 docs/snippets/tag_forms/external.rst create mode 100644 docs/snippets/tag_forms/internal.rst diff --git a/docs/snippets/tag_forms/adjacent.rst b/docs/snippets/tag_forms/adjacent.rst new file mode 100644 index 0000000..4131394 --- /dev/null +++ b/docs/snippets/tag_forms/adjacent.rst @@ -0,0 +1,9 @@ +.. code-block:: json + + { + "#type": "cat", + "#value": { + "name": "Max", + "color": "tuxedo", + } + } diff --git a/docs/snippets/tag_forms/external.rst b/docs/snippets/tag_forms/external.rst new file mode 100644 index 0000000..fba0d6f --- /dev/null +++ b/docs/snippets/tag_forms/external.rst @@ -0,0 +1,8 @@ +.. code-block:: json + + { + "cat": { + "name": "Max", + "color": "tuxedo", + } + } diff --git a/docs/snippets/tag_forms/internal.rst b/docs/snippets/tag_forms/internal.rst new file mode 100644 index 0000000..1fde5b5 --- /dev/null +++ b/docs/snippets/tag_forms/internal.rst @@ -0,0 +1,7 @@ +.. code-block:: json + + { + "#type": "cat", + "name": "Max", + "color": "tuxedo", + } From 82f92be4f319d8fced8d1f9f109d47a18c6e4497 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 24 Aug 2021 09:49:04 -0400 Subject: [PATCH 44/53] black --- tests/test_fields.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_fields.py b/tests/test_fields.py index e60e53a..61dc17b 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -388,6 +388,8 @@ class C: assert schema.dumps(objects) == serialized assert schema.loads(serialized) == objects + + # end tagged_union_example From 2b8ca427fc246428f681756100313bc8b12d7ab4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 24 Aug 2021 10:09:07 -0400 Subject: [PATCH 45/53] doc warning tidy --- docs/reference/fields.rst | 21 +++++++++++++++++++++ src/desert/_fields.py | 16 ++++++++-------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/reference/fields.rst b/docs/reference/fields.rst index c08307b..4542f98 100644 --- a/docs/reference/fields.rst +++ b/docs/reference/fields.rst @@ -76,6 +76,22 @@ The helpers below create :class:`desert._fields.TaggedUnionField` instances that :undoc-members: :show-inheritance: +The fields can be created from :class:`desert._fields.FromObjectProtocol` and :class:`desert._fields.FromTagProtocol` instead of registries, if need. + +.. autofunction:: desert._fields.adjacently_tagged_union +.. autofunction:: desert._fields.internally_tagged_union +.. autofunction:: desert._fields.externally_tagged_union + +.. autoclass:: desert._fields.FromObjectProtocol + :members: __call__ + :undoc-members: + :show-inheritance: + +.. autoclass:: desert._fields.FromTagProtocol + :members: __call__ + :undoc-members: + :show-inheritance: + .. _tagged_union_registries: @@ -101,3 +117,8 @@ As an example, you might register a custom class ``Cat`` by providing a hint of :members: :undoc-members: :show-inheritance: + +.. autoclass:: desert._fields.HintTagField + :members: + :undoc-members: + :show-inheritance: diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 93f95e2..8504e2b 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -209,19 +209,19 @@ class TaggedUnionField(marshmallow.fields.Field): using helper functions to fill out the needed functions in a consistent manner. Helpers are provided both to directly create various forms of this field as well - as to create the same from a :class:`FieldRegistry`. + as to create the same from a :class:`FieldRegistryProtocol`. - From a registry - - :func:`externally_tagged_union_from_registry` - - :func:`internally_tagged_union_from_registry` - :func:`adjacently_tagged_union_from_registry` + - :func:`internally_tagged_union_from_registry` + - :func:`externally_tagged_union_from_registry` - Direct - - :func:`externally_tagged_union` - - :func:`internally_tagged_union` - :func:`adjacently_tagged_union` + - :func:`internally_tagged_union` + - :func:`externally_tagged_union` """ def __init__( @@ -312,7 +312,7 @@ def externally_tagged_union( def externally_tagged_union_from_registry( registry: FieldRegistryProtocol, ) -> TaggedUnionField: - """Use a :class:`FieldRegistry` to create a :class:`TaggedUnionField` that supports + """Use a :class:`FieldRegistryProtocol` to create a :class:`TaggedUnionField` that supports the externally tagged form. Externally tagged data has the following form. .. include:: ../snippets/tag_forms/external.rst @@ -368,7 +368,7 @@ def internally_tagged_union_from_registry( registry: FieldRegistryProtocol, type_key: str = default_tagged_type_key, ) -> TaggedUnionField: - """Use a :class:`FieldRegistry` to create a :class:`TaggedUnionField` that supports + """Use a :class:`FieldRegistryProtocol` to create a :class:`TaggedUnionField` that supports the internally tagged form. Internally tagged data has the following form. .. include:: ../snippets/tag_forms/internal.rst @@ -428,7 +428,7 @@ def adjacently_tagged_union_from_registry( type_key: str = default_tagged_type_key, value_key: str = default_tagged_value_key, ) -> TaggedUnionField: - """Use a :class:`FieldRegistry` to create a :class:`TaggedUnionField` that supports + """Use a :class:`FieldRegistryProtocol` to create a :class:`TaggedUnionField` that supports the adjacently tagged form. Adjacently tagged data has the following form. .. include:: ../snippets/tag_forms/adjacent.rst From c256d807b8e54945779067ac2f860d77a1c3f8fe Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 24 Aug 2021 11:25:11 -0400 Subject: [PATCH 46/53] docutils < 0.17 for circular include error --- docs/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 99b8ca6..0782025 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,7 @@ sphinx~=4.1 # >= 1.0.0rc1 for https://github.com/readthedocs/sphinx_rtd_theme/issues/1115 sphinx-rtd-theme >= 1.0.0rc1 +# < 0.17 for /home/docs/checkouts/readthedocs.org/user_builds/desert/checkouts/94/src/desert/_fields.py:docstring of desert._fields.externally_tagged_union_from_registry:4: WARNING: circular inclusion in "include" directive: snippets/tag_forms/external.rst < snippets/tag_forms/internal.rst < snippets/tag_forms/adjacent.rst < snippets/tag_forms/external.rst < reference/fields.rst +docutils < 0.17 sphinx-autodoc-typehints -e .[dev] From dc5d731bff1c1e8e275cc57990a1215743004076 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 24 Aug 2021 11:32:35 -0400 Subject: [PATCH 47/53] avoid trailing blank lines in example in docs --- tests/test_fields.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index 61dc17b..a42f4da 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -388,9 +388,7 @@ class C: assert schema.dumps(objects) == serialized assert schema.loads(serialized) == objects - - -# end tagged_union_example + # end tagged_union_example def test_raises_for_tag_reregistration() -> None: From bd64aa02b49d36926835c874b490632a2a14fa1d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 24 Aug 2021 11:43:46 -0400 Subject: [PATCH 48/53] xfail for working-.__origin__-requiring tests on < 3.7 --- tests/test_fields.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_fields.py b/tests/test_fields.py index a42f4da..d57d6bc 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2,6 +2,7 @@ import collections.abc import decimal import json +import sys import typing as t # https://github.com/pytest-dev/pytest/issues/7469 @@ -28,6 +29,7 @@ class ExampleData: field: marshmallow.fields.Field # TODO: can we be more specific? hint: t.Any + requires_origin: bool = False @classmethod def build( @@ -36,6 +38,7 @@ def build( to_serialize: object, tag: str, field: marshmallow.fields.Field, + requires_origin: bool = False, serialized: object = _NOTHING, deserialized: object = _NOTHING, ) -> "ExampleData": @@ -52,6 +55,7 @@ def build( deserialized=deserialized, tag=tag, field=field, + requires_origin=requires_origin, ) @@ -86,6 +90,7 @@ def build( to_serialize=["abc", "2", "mno"], tag="string_list_tag", field=marshmallow.fields.List(marshmallow.fields.String()), + requires_origin=True, ), ExampleData.build( hint=t.Sequence[str], @@ -268,6 +273,9 @@ def test_externally_tagged_serialize( example_data: ExampleData, externally_tagged_field: desert._fields.TaggedUnionField, ) -> None: + if example_data.requires_origin and sys.version_info < (3, 7): + pytest.xfail() + obj = {"key": example_data.to_serialize} serialized = externally_tagged_field.serialize("key", obj) @@ -360,6 +368,9 @@ def test_adjacently_tagged_serialize( example_data: ExampleData, adjacently_tagged_field: desert._fields.TaggedUnionField, ) -> None: + if example_data.requires_origin and sys.version_info < (3, 7): + pytest.xfail() + obj = {"key": example_data.to_serialize} serialized = adjacently_tagged_field.serialize("key", obj) From f8a377f04d3665c16d1cf879701151cf95752b39 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 25 Aug 2021 11:52:22 -0400 Subject: [PATCH 49/53] parametrize against *_tagged_union[_from_registry] for coverage --- tests/test_fields.py | 54 ++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index d57d6bc..0c0a0e3 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -232,14 +232,22 @@ def test_registry_raises_for_multiple_matches() -> None: registry.from_object(value=[]) -@pytest.fixture(name="externally_tagged_field") +@pytest.fixture(name="externally_tagged_field", params=[False, True]) def _externally_tagged_field( + request: _pytest.fixtures.SubRequest, registry: desert._fields.FieldRegistryProtocol, ) -> desert._fields.TaggedUnionField: - return desert._fields.externally_tagged_union( - from_object=registry.from_object, - from_tag=registry.from_tag, - ) + field: desert._fields.TaggedUnionField + + if request.param: + field = desert._fields.externally_tagged_union_from_registry(registry=registry) + else: + field = desert._fields.externally_tagged_union( + from_object=registry.from_object, + from_tag=registry.from_tag, + ) + + return field def test_externally_tagged_deserialize( @@ -283,14 +291,22 @@ def test_externally_tagged_serialize( assert serialized == {example_data.tag: example_data.serialized} -@pytest.fixture(name="internally_tagged_field") +@pytest.fixture(name="internally_tagged_field", params=[False, True]) def _internally_tagged_field( + request: _pytest.fixtures.SubRequest, registry: desert._fields.FieldRegistryProtocol, ) -> desert._fields.TaggedUnionField: - return desert._fields.internally_tagged_union( - from_object=registry.from_object, - from_tag=registry.from_tag, - ) + field: desert._fields.TaggedUnionField + + if request.param: + field = desert._fields.internally_tagged_union_from_registry(registry=registry) + else: + field = desert._fields.internally_tagged_union( + from_object=registry.from_object, + from_tag=registry.from_tag, + ) + + return field def test_to_internally_tagged_raises_for_tag_collision() -> None: @@ -327,14 +343,22 @@ def test_internally_tagged_serialize( } -@pytest.fixture(name="adjacently_tagged_field") +@pytest.fixture(name="adjacently_tagged_field", params=[False, True]) def _adjacently_tagged_field( + request: _pytest.fixtures.SubRequest, registry: desert._fields.FieldRegistryProtocol, ) -> desert._fields.TaggedUnionField: - return desert._fields.adjacently_tagged_union( - from_object=registry.from_object, - from_tag=registry.from_tag, - ) + field: desert._fields.TaggedUnionField + + if request.param: + field = desert._fields.adjacently_tagged_union_from_registry(registry=registry) + else: + field = desert._fields.adjacently_tagged_union( + from_object=registry.from_object, + from_tag=registry.from_tag, + ) + + return field def test_adjacently_tagged_deserialize( From 694000f87abbfe262401c299de9b6f0d01c17d37 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 30 Aug 2021 19:29:24 -0400 Subject: [PATCH 50/53] actually test tagged union examples --- dev-requirements.txt | 4 +- docs/reference/fields.rst | 28 ++--- docs/snippets/tag_forms/adjacent.rst | 9 -- docs/snippets/tag_forms/external.rst | 8 -- docs/snippets/tag_forms/internal.rst | 7 -- src/desert/_fields.py | 9 +- test-requirements.in | 1 + test-requirements.txt | 5 +- tests/test_fields.py | 172 ++++++++++++++++++++++++++- 9 files changed, 194 insertions(+), 49 deletions(-) delete mode 100644 docs/snippets/tag_forms/adjacent.rst delete mode 100644 docs/snippets/tag_forms/external.rst delete mode 100644 docs/snippets/tag_forms/internal.rst diff --git a/dev-requirements.txt b/dev-requirements.txt index d186200..5240495 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -235,7 +235,9 @@ importlib-metadata==4.6.1 \ importlib-resources==5.2.2 \ --hash=sha256:2480d8e07d1890056cb53c96e3de44fead9c62f2ba949b0f2e4c4345f4afa977 \ --hash=sha256:a65882a4d0fe5fbf702273456ba2ce74fe44892c25e42e057aca526b702a6d4b - # via virtualenv + # via + # -r test-requirements.in + # virtualenv incremental==21.3.0 \ --hash=sha256:02f5de5aff48f6b9f665d99d48bfc7ec03b6e3943210de7cfc88856d755d6f57 \ --hash=sha256:92014aebc6a20b78a8084cdd5645eeaa7f74b8933f70fa3ada2cfbd1e3b54321 diff --git a/docs/reference/fields.rst b/docs/reference/fields.rst index 4542f98..b0b85de 100644 --- a/docs/reference/fields.rst +++ b/docs/reference/fields.rst @@ -22,38 +22,38 @@ A tag indicating the object's type can be applied in various ways. Presently three forms are implemented: adjacently tagged, internally tagged, and externally tagged. Adjacently tagged is the most explicit form and is the recommended default. You can write your own helper functions to implement your own tagging form if needed and still make use of the rest of the mechanisms implemented here. +A code example follows the forms below and provides related reference. - A class definition and bare serialized object for reference - .. code-block:: python + .. literalinclude:: ../../tests/test_fields.py + :language: python + :start-after: # start cat_class_example + :end-before: # end cat_class_example - @dataclasses.dataclass - class Cat: - name: str - color: str + .. literalinclude:: ../../tests/example/untagged.json + :language: json - .. code-block:: json - - { - "name": "Max", - "color": "tuxedo", - } - Adjacently tagged - .. include:: ../snippets/tag_forms/adjacent.rst + .. literalinclude:: ../../tests/example/adjacent.json + :language: json - Internally tagged - .. include:: ../snippets/tag_forms/internal.rst + .. literalinclude:: ../../tests/example/internal.json + :language: json - Externally tagged - .. include:: ../snippets/tag_forms/external.rst + .. literalinclude:: ../../tests/example/external.json + :language: json The code below is an actual test from the Desert test suite that provides an example usage of the tools that will be covered in detail below. .. literalinclude:: ../../tests/test_fields.py + :language: python :start-after: # start tagged_union_example :end-before: # end tagged_union_example diff --git a/docs/snippets/tag_forms/adjacent.rst b/docs/snippets/tag_forms/adjacent.rst deleted file mode 100644 index 4131394..0000000 --- a/docs/snippets/tag_forms/adjacent.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. code-block:: json - - { - "#type": "cat", - "#value": { - "name": "Max", - "color": "tuxedo", - } - } diff --git a/docs/snippets/tag_forms/external.rst b/docs/snippets/tag_forms/external.rst deleted file mode 100644 index fba0d6f..0000000 --- a/docs/snippets/tag_forms/external.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. code-block:: json - - { - "cat": { - "name": "Max", - "color": "tuxedo", - } - } diff --git a/docs/snippets/tag_forms/internal.rst b/docs/snippets/tag_forms/internal.rst deleted file mode 100644 index 1fde5b5..0000000 --- a/docs/snippets/tag_forms/internal.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. code-block:: json - - { - "#type": "cat", - "name": "Max", - "color": "tuxedo", - } diff --git a/src/desert/_fields.py b/src/desert/_fields.py index 8504e2b..8bb59ae 100644 --- a/src/desert/_fields.py +++ b/src/desert/_fields.py @@ -315,7 +315,8 @@ def externally_tagged_union_from_registry( """Use a :class:`FieldRegistryProtocol` to create a :class:`TaggedUnionField` that supports the externally tagged form. Externally tagged data has the following form. - .. include:: ../snippets/tag_forms/external.rst + .. literalinclude:: ../../tests/example/external.json + :language: json """ return externally_tagged_union( @@ -371,7 +372,8 @@ def internally_tagged_union_from_registry( """Use a :class:`FieldRegistryProtocol` to create a :class:`TaggedUnionField` that supports the internally tagged form. Internally tagged data has the following form. - .. include:: ../snippets/tag_forms/internal.rst + .. literalinclude:: ../../tests/example/internal.json + :language: json """ return internally_tagged_union( @@ -431,7 +433,8 @@ def adjacently_tagged_union_from_registry( """Use a :class:`FieldRegistryProtocol` to create a :class:`TaggedUnionField` that supports the adjacently tagged form. Adjacently tagged data has the following form. - .. include:: ../snippets/tag_forms/adjacent.rst + .. literalinclude:: ../../tests/example/adjacent.json + :language: json """ return adjacently_tagged_union( diff --git a/test-requirements.in b/test-requirements.in index 1b5ab61..2213c53 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -1,5 +1,6 @@ coverage cuvner +importlib_resources marshmallow-enum marshmallow-union pytest diff --git a/test-requirements.txt b/test-requirements.txt index 519efb2..33afc4a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -100,7 +100,6 @@ importlib-metadata==4.6.1 \ --hash=sha256:9f55f560e116f8643ecf2922d9cd3e1c7e8d52e683178fecd9d08f6aa357e11e # via # -r test-requirements.in - # backports.entry-points-selectable # click # pluggy # pytest @@ -109,7 +108,9 @@ importlib-metadata==4.6.1 \ importlib-resources==5.2.2 \ --hash=sha256:2480d8e07d1890056cb53c96e3de44fead9c62f2ba949b0f2e4c4345f4afa977 \ --hash=sha256:a65882a4d0fe5fbf702273456ba2ce74fe44892c25e42e057aca526b702a6d4b - # via virtualenv + # via + # -r test-requirements.in + # virtualenv incremental==21.3.0 \ --hash=sha256:02f5de5aff48f6b9f665d99d48bfc7ec03b6e3943210de7cfc88856d755d6f57 \ --hash=sha256:92014aebc6a20b78a8084cdd5645eeaa7f74b8933f70fa3ada2cfbd1e3b54321 diff --git a/tests/test_fields.py b/tests/test_fields.py index 0c0a0e3..5006ac2 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,5 +1,6 @@ import abc import collections.abc +import dataclasses import decimal import json import sys @@ -8,12 +9,15 @@ # https://github.com/pytest-dev/pytest/issues/7469 import _pytest.fixtures import attr +import importlib_resources import marshmallow import pytest +import typing_extensions import desert._fields import desert.exceptions +import tests.example # TODO: test that field constructor doesn't tromple Field parameters @@ -402,8 +406,10 @@ def test_adjacently_tagged_serialize( assert serialized == {"#type": example_data.tag, "#value": example_data.serialized} -# start tagged_union_example -def test_actual_example() -> None: +@pytest.mark.parametrize( + argnames=["type_string", "value"], argvalues=[["str", "3"], ["int", 7]] +) +def test_actual_example(type_string: str, value: t.Union[int, str]) -> None: registry = desert._fields.TypeAndHintFieldRegistry() registry.register(hint=str, tag="str", field=marshmallow.fields.String()) registry.register(hint=int, tag="int", field=marshmallow.fields.Integer()) @@ -417,13 +423,12 @@ class C: schema = desert.schema(C) - objects = C(union="3") - marshalled = {"union": {"#type": "str", "#value": "3"}} + objects = C(union=value) + marshalled = {"union": {"#type": type_string, "#value": value}} serialized = json.dumps(marshalled) assert schema.dumps(objects) == serialized assert schema.loads(serialized) == objects - # end tagged_union_example def test_raises_for_tag_reregistration() -> None: @@ -434,3 +439,160 @@ def test_raises_for_tag_reregistration() -> None: registry.register( hint=int, tag="duplicate_tag", field=marshmallow.fields.Integer() ) + + +# start cat_class_example +@dataclasses.dataclass +class Cat: + name: str + color: str + # end cat_class_example + + +def test_untagged_serializes_like_snippet() -> None: + cat = Cat(name="Max", color="tuxedo") + + reference = importlib_resources.read_text(tests.example, "untagged.json").strip() + + schema = desert.schema(Cat, meta={"ordered": True}) + dumped = schema.dumps(cat, indent=4) + + assert dumped == reference + + +# Marshmallow fields expect to serialize an attribute, not an object directly. +# This class gives us somewhere to stick the object of interest to make the field +# happy. +@attr.frozen +class CatCarrier: + an_object: Cat + + +class FromRegistryProtocol(typing_extensions.Protocol): + def __call__( + self, registry: desert._fields.FieldRegistryProtocol + ) -> desert._fields.TaggedUnionField: + ... + + +@attr.frozen +class ResourceAndRegistryFunction: + resource_name: str + from_registry_function: FromRegistryProtocol + + +@pytest.fixture( + name="resource_and_registry_function", + params=[ + ResourceAndRegistryFunction( + resource_name="adjacent.json", + from_registry_function=desert._fields.adjacently_tagged_union_from_registry, + ), + ResourceAndRegistryFunction( + resource_name="internal.json", + from_registry_function=desert._fields.internally_tagged_union_from_registry, + ), + ResourceAndRegistryFunction( + resource_name="external.json", + from_registry_function=desert._fields.externally_tagged_union_from_registry, + ), + ], +) +def resource_and_registry_function_fixture( + request: _pytest.fixtures.SubRequest, +) -> ResourceAndRegistryFunction: + return request.param # type: ignore[no-any-return] + + +def test_tagged_serializes_like_snippet( + resource_and_registry_function: ResourceAndRegistryFunction, +) -> None: + cat = Cat(name="Max", color="tuxedo") + + registry = desert._fields.TypeAndHintFieldRegistry() + registry.register( + hint=Cat, + tag="cat", + field=marshmallow.fields.Nested(desert.schema(Cat, meta={"ordered": True})), + ) + + reference = importlib_resources.read_text( + tests.example, resource_and_registry_function.resource_name + ).strip() + + field = resource_and_registry_function.from_registry_function(registry=registry) + marshalled = field.serialize(attr="an_object", obj=CatCarrier(an_object=cat)) + dumped = json.dumps(marshalled, indent=4) + + assert dumped == reference + + +def test_tagged_deserializes_from_snippet( + resource_and_registry_function: ResourceAndRegistryFunction, +) -> None: + registry = desert._fields.TypeAndHintFieldRegistry() + registry.register( + hint=Cat, + tag="cat", + field=marshmallow.fields.Nested(desert.schema(Cat, meta={"ordered": True})), + ) + + reference = importlib_resources.read_text( + tests.example, resource_and_registry_function.resource_name + ).strip() + + field = resource_and_registry_function.from_registry_function(registry=registry) + deserialized_cat = field.deserialize(value=json.loads(reference)) + + assert deserialized_cat == Cat(name="Max", color="tuxedo") + + +# start tagged_union_example +def test_tagged_union_example() -> None: + @dataclasses.dataclass + class Dog: + name: str + color: str + + registry = desert._fields.TypeAndHintFieldRegistry() + registry.register( + hint=Cat, + tag="cat", + field=marshmallow.fields.Nested(desert.schema(Cat, meta={"ordered": True})), + ) + registry.register( + hint=Dog, + tag="dog", + field=marshmallow.fields.Nested(desert.schema(Dog, meta={"ordered": True})), + ) + + field = desert._fields.adjacently_tagged_union_from_registry(registry=registry) + + @dataclasses.dataclass + class CatsAndDogs: + union: t.Union[Cat, Dog] = desert.field(marshmallow_field=field) + + schema = desert.schema(CatsAndDogs) + + with_a_cat = CatsAndDogs(union=Cat(name="Max", color="tuxedo")) + with_a_dog = CatsAndDogs(union=Dog(name="Bubbles", color="black spots on white")) + + marshalled_cat = { + "union": {"#type": "cat", "#value": {"name": "Max", "color": "tuxedo"}} + } + marshalled_dog = { + "union": { + "#type": "dog", + "#value": {"name": "Bubbles", "color": "black spots on white"}, + } + } + + dumped_cat = json.dumps(marshalled_cat) + dumped_dog = json.dumps(marshalled_dog) + + assert dumped_cat == schema.dumps(with_a_cat) + assert dumped_dog == schema.dumps(with_a_dog) + + assert with_a_cat == schema.loads(dumped_cat) + assert with_a_dog == schema.loads(dumped_dog) + # end tagged_union_example From d5acb6398ced791d77ed0036dbf9c894def9ce74 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 30 Aug 2021 19:43:22 -0400 Subject: [PATCH 51/53] add new example resources --- tests/example/__init__.py | 0 tests/example/adjacent.json | 7 +++++++ tests/example/external.json | 6 ++++++ tests/example/internal.json | 5 +++++ tests/example/untagged.json | 4 ++++ 5 files changed, 22 insertions(+) create mode 100644 tests/example/__init__.py create mode 100644 tests/example/adjacent.json create mode 100644 tests/example/external.json create mode 100644 tests/example/internal.json create mode 100644 tests/example/untagged.json diff --git a/tests/example/__init__.py b/tests/example/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/example/adjacent.json b/tests/example/adjacent.json new file mode 100644 index 0000000..a67c923 --- /dev/null +++ b/tests/example/adjacent.json @@ -0,0 +1,7 @@ +{ + "#type": "cat", + "#value": { + "name": "Max", + "color": "tuxedo" + } +} diff --git a/tests/example/external.json b/tests/example/external.json new file mode 100644 index 0000000..bcf8c5d --- /dev/null +++ b/tests/example/external.json @@ -0,0 +1,6 @@ +{ + "cat": { + "name": "Max", + "color": "tuxedo" + } +} diff --git a/tests/example/internal.json b/tests/example/internal.json new file mode 100644 index 0000000..38d3cd3 --- /dev/null +++ b/tests/example/internal.json @@ -0,0 +1,5 @@ +{ + "#type": "cat", + "name": "Max", + "color": "tuxedo" +} diff --git a/tests/example/untagged.json b/tests/example/untagged.json new file mode 100644 index 0000000..a0707b5 --- /dev/null +++ b/tests/example/untagged.json @@ -0,0 +1,4 @@ +{ + "name": "Max", + "color": "tuxedo" +} From f6d23148c53e19423addd8dbdf5d807a4e4e8467 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 30 Aug 2021 19:46:50 -0400 Subject: [PATCH 52/53] isort --- tests/test_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index 5006ac2..9b883e9 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -16,9 +16,9 @@ import desert._fields import desert.exceptions - import tests.example + # TODO: test that field constructor doesn't tromple Field parameters _NOTHING = object() From ea773b7bc7df6591ff7c0e538e2f96a11637657b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 8 Sep 2021 17:32:46 -0400 Subject: [PATCH 53/53] CatsAndDogs corrected to CatOrDog --- tests/test_fields.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index 9b883e9..de1e4a7 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -569,13 +569,13 @@ class Dog: field = desert._fields.adjacently_tagged_union_from_registry(registry=registry) @dataclasses.dataclass - class CatsAndDogs: + class CatOrDog: union: t.Union[Cat, Dog] = desert.field(marshmallow_field=field) - schema = desert.schema(CatsAndDogs) + schema = desert.schema(CatOrDog) - with_a_cat = CatsAndDogs(union=Cat(name="Max", color="tuxedo")) - with_a_dog = CatsAndDogs(union=Dog(name="Bubbles", color="black spots on white")) + with_a_cat = CatOrDog(union=Cat(name="Max", color="tuxedo")) + with_a_dog = CatOrDog(union=Dog(name="Bubbles", color="black spots on white")) marshalled_cat = { "union": {"#type": "cat", "#value": {"name": "Max", "color": "tuxedo"}}