From 606c50c7d9d86a6782645bd5b0288f062dafaf1b Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 00:32:27 +0300 Subject: [PATCH 01/37] Convert docstrings to double quotes --- dpath/segments.py | 72 +++++++++++++++++++++--------------------- dpath/util.py | 36 ++++++++++----------- tests/test_segments.py | 60 +++++++++++++++++------------------ 3 files changed, 84 insertions(+), 84 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 1016a06..07efd04 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -5,11 +5,11 @@ def kvs(node): - ''' + """ Return a (key, value) iterator for the node. kvs(node) -> (generator -> (key, value)) - ''' + """ try: return iter(node.items()) except AttributeError: @@ -23,23 +23,23 @@ def kvs(node): def leaf(thing): - ''' + """ Return True if thing is a leaf, otherwise False. leaf(thing) -> bool - ''' + """ leaves = (bytes, str, int, float, bool, type(None)) return isinstance(thing, leaves) def leafy(thing): - ''' + """ Same as leaf(thing), but also treats empty sequences and dictionaries as True. leafy(thing) -> bool - ''' + """ try: return leaf(thing) or len(thing) == 0 @@ -49,12 +49,12 @@ def leafy(thing): def walk(obj, location=()): - ''' + """ Yield all valid (segments, value) pairs (from a breadth-first search, right-to-left on sequences). walk(obj) -> (generator -> (segments, value)) - ''' + """ if not leaf(obj): for k, v in kvs(obj): length = None @@ -75,11 +75,11 @@ def walk(obj, location=()): def get(obj, segments): - ''' + """ Return the value at the path indicated by segments. get(obj, segments) -> value - ''' + """ current = obj for (i, segment) in enumerate(segments): if leaf(current): @@ -90,11 +90,11 @@ def get(obj, segments): def has(obj, segments): - ''' + """ Return True if the path exists in the obj. Otherwise return False. has(obj, segments) -> bool - ''' + """ try: get(obj, segments) return True @@ -103,12 +103,12 @@ def has(obj, segments): def expand(segments): - ''' + """ Yield a tuple of segments for each possible length of segments. Starting from the shortest length of segments and increasing by 1. expand(keys) -> (..., keys[:-2], keys[:-1]) - ''' + """ index = 0 for segment in segments: index += 1 @@ -116,11 +116,11 @@ def expand(segments): def types(obj, segments): - ''' + """ For each segment produce a tuple of (segment, type(value)). types(obj, segments) -> ((segment[0], type0), (segment[1], type1), ...) - ''' + """ result = [] for depth in expand(segments): result.append((depth[-1], type(get(obj, depth)))) @@ -128,31 +128,31 @@ def types(obj, segments): def leaves(obj): - ''' + """ Yield all leaves as (segment, value) pairs. leaves(obj) -> (generator -> (segment, value)) - ''' + """ return filter(lambda p: leafy(p[1]), walk(obj)) def int_str(segment): - ''' + """ If the segment is an integer, return the string conversion. Otherwise return the segment unchanged. The conversion uses 'str'. int_str(segment) -> str - ''' + """ if isinstance(segment, int): return str(segment) return segment class Star(object): - ''' + """ Used to create a global STAR symbol for tracking stars added when expanding star-star globs. - ''' + """ pass @@ -160,7 +160,7 @@ class Star(object): def match(segments, glob): - ''' + """ Return True if the segments match the given glob, otherwise False. For the purposes of matching, integers are converted to their string @@ -177,7 +177,7 @@ def match(segments, glob): throws an exception the result will be False. match(segments, glob) -> bool - ''' + """ segments = tuple(segments) glob = tuple(glob) @@ -238,12 +238,12 @@ def match(segments, glob): def extend(thing, index, value=None): - ''' + """ Extend a sequence like thing such that it contains at least index + 1 many elements. The extension values will be None (default). extend(thing, int) -> [thing..., None, ...] - ''' + """ try: expansion = (type(thing)()) @@ -263,12 +263,12 @@ def extend(thing, index, value=None): def __default_creator__(current, segments, i, hints=()): - ''' + """ Create missing path components. If the segment is an int, then it will create a list. Otherwise a dictionary is created. set(obj, segments, value) -> obj - ''' + """ segment = segments[i] length = len(segments) @@ -293,13 +293,13 @@ def __default_creator__(current, segments, i, hints=()): def set(obj, segments, value, creator=__default_creator__, hints=()): - ''' + """ Set the value in obj at the place indicated by segments. If creator is not None (default __default_creator__), then call the creator function to create any missing path components. set(obj, segments, value) -> obj - ''' + """ current = obj length = len(segments) @@ -331,7 +331,7 @@ def set(obj, segments, value, creator=__default_creator__, hints=()): def fold(obj, f, acc): - ''' + """ Walk obj applying f to each path and returning accumulator acc. The function f will be called, for each result in walk(obj): @@ -343,7 +343,7 @@ def fold(obj, f, acc): retrieved from the walk. fold(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc - ''' + """ for pair in walk(obj): if f(obj, pair, acc) is False: break @@ -351,14 +351,14 @@ def fold(obj, f, acc): def foldm(obj, f, acc): - ''' + """ Same as fold(), but permits mutating obj. This requires all paths in walk(obj) to be loaded into memory (whereas fold does not). foldm(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc - ''' + """ pairs = tuple(walk(obj)) for pair in pairs: (segments, value) = pair @@ -368,13 +368,13 @@ def foldm(obj, f, acc): def view(obj, glob): - ''' + """ Return a view of the object where the glob matches. A view retains the same form as the obj, but is limited to only the paths that matched. Views are new objects (a deepcopy of the matching values). view(obj, glob) -> obj' - ''' + """ def f(obj, pair, result): (segments, value) = pair if match(segments, glob): diff --git a/dpath/util.py b/dpath/util.py index 48c03be..39aaa6e 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -11,7 +11,7 @@ def __safe_path__(path, separator): - ''' + """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -19,7 +19,7 @@ def __safe_path__(path, separator): separator stripped off. If you pass a list path, the separator is ignored, and is assumed to be part of each key glob. It will not be stripped. - ''' + """ if not dpath.segments.leaf(path): segments = path else: @@ -45,7 +45,7 @@ def __safe_path__(path, separator): def new(obj, path, value, separator='/', creator=None): - ''' + """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only change existing keys). @@ -57,7 +57,7 @@ def new(obj, path, value, separator='/', creator=None): creator allows you to pass in a creator method that is responsible for creating missing keys at arbitrary levels of the path (see the help for dpath.path.set) - ''' + """ segments = __safe_path__(path, separator) if creator: return dpath.segments.set(obj, segments, value, creator=creator) @@ -65,12 +65,12 @@ def new(obj, path, value, separator='/', creator=None): def delete(obj, glob, separator='/', afilter=None): - ''' + """ Given a obj, delete all elements that match the glob. Returns the number of deleted objects. Raises PathNotFound if no paths are found to delete. - ''' + """ globlist = __safe_path__(glob, separator) def f(obj, pair, counter): @@ -122,10 +122,10 @@ def f(obj, pair, counter): def set(obj, glob, value, separator='/', afilter=None): - ''' + """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. - ''' + """ globlist = __safe_path__(glob, separator) def f(obj, pair, counter): @@ -147,7 +147,7 @@ def f(obj, pair, counter): def get(obj, glob, separator='/', default=_DEFAULT_SENTINAL): - ''' + """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. If the glob is not found and a default is provided, @@ -155,7 +155,7 @@ def get(obj, glob, separator='/', default=_DEFAULT_SENTINAL): If more than one leaf matches the glob, ValueError is raised. If the glob is not found and a default is not provided, KeyError is raised. - ''' + """ if glob == '/': return obj @@ -183,32 +183,32 @@ def f(obj, pair, results): def values(obj, glob, separator='/', afilter=None, dirs=True): - ''' + """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). - ''' + """ yielded = True return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): - ''' + """ Given a path glob, return a dictionary containing all keys that matched the given glob. If 'yielded' is true, then a dictionary will not be returned. Instead tuples will be yielded in the form of (path, value) for every element in the document that matched the glob. - ''' + """ globlist = __safe_path__(glob, separator) def keeper(segments, found): - ''' + """ Generalized test for use in both yielded and folded cases. Returns True if we want this result. Otherwise returns False. - ''' + """ if not dirs and not dpath.segments.leaf(found): return False @@ -234,7 +234,7 @@ def f(obj, pair, result): def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): - ''' + """ Merge source into destination. Like dict.update() but performs deep merging. @@ -272,7 +272,7 @@ def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): * MERGE_TYPESAFE : When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source replaces the destination in this situation. - ''' + """ filtered_src = search(src, '**', afilter=afilter, separator='/') def are_both_mutable(o1, o2): diff --git a/tests/test_segments.py b/tests/test_segments.py index af9df85..2fdf070 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -39,36 +39,36 @@ def teardown(): @given(random_node) def test_kvs(node): - ''' + """ Given a node, kvs should produce a key that when used to extract from the node renders the exact same value given. - ''' + """ for k, v in api.kvs(node): assert node[k] is v @given(random_leaf) def test_leaf_with_leaf(leaf): - ''' + """ Given a leaf, leaf should return True. - ''' + """ assert api.leaf(leaf) is True @given(random_node) def test_leaf_with_node(node): - ''' + """ Given a node, leaf should return False. - ''' + """ assert api.leaf(node) is False @given(random_thing) def test_walk(thing): - ''' + """ Given a thing to walk, walk should yield key, value pairs where key is a tuple of non-zero length. - ''' + """ for k, v in api.walk(thing): assert isinstance(k, tuple) assert len(k) > 0 @@ -76,19 +76,19 @@ def test_walk(thing): @given(random_node) def test_get(node): - ''' + """ Given a node, get should return the exact value given a key for all key, value pairs in the node. - ''' + """ for k, v in api.walk(node): assert api.get(node, k) is v @given(random_node) def test_has(node): - ''' + """ Given a node, has should return True for all paths, False otherwise. - ''' + """ for k, v in api.walk(node): assert api.has(node, k) is True @@ -100,10 +100,10 @@ def test_has(node): @given(random_segments) def test_expand(segments): - ''' + """ Given segments expand should produce as many results are there were segments and the last result should equal the given segments. - ''' + """ count = len(segments) result = list(api.expand(segments)) @@ -115,10 +115,10 @@ def test_expand(segments): @given(random_node) def test_types(node): - ''' + """ Given a node, types should yield a tuple of key, type pairs and the type indicated should equal the type of the value. - ''' + """ for k, v in api.walk(node): ts = api.types(node, k) ta = () @@ -129,9 +129,9 @@ def test_types(node): @given(random_node) def test_leaves(node): - ''' + """ Given a node, leaves should yield only leaf key, value pairs. - ''' + """ for k, v in api.leaves(node): assert api.leafy(v) @@ -245,18 +245,18 @@ def random_segments_with_nonmatching_glob(draw): @given(random_segments_with_glob()) def test_match(pair): - ''' + """ Given segments and a known good glob, match should be True. - ''' + """ (segments, glob) = pair assert api.match(segments, glob) is True @given(random_segments_with_nonmatching_glob()) def test_match_nonmatching(pair): - ''' + """ Given segments and a known bad glob, match should be False. - ''' + """ print(pair) (segments, glob) = pair assert api.match(segments, glob) is False @@ -280,9 +280,9 @@ def random_leaves(draw): @given(walkable=random_walk(), value=random_thing) def test_set_walkable(walkable, value): - ''' + """ Given a walkable location, set should be able to update any value. - ''' + """ (node, (segments, found)) = walkable api.set(node, segments, value) assert api.get(node, segments) is value @@ -294,10 +294,10 @@ def test_set_walkable(walkable, value): value=random_thing, extension=random_segments) def test_set_create_missing(walkable, kstr, kint, value, extension): - ''' + """ Given a walkable non-leaf, set should be able to create missing nodes and set a new value. - ''' + """ (node, (segments, found)) = walkable assume(api.leaf(found)) @@ -319,9 +319,9 @@ def test_set_create_missing(walkable, kstr, kint, value, extension): @given(thing=random_thing) def test_fold(thing): - ''' + """ Given a thing, count paths with fold. - ''' + """ def f(o, p, a): a[0] += 1 @@ -331,9 +331,9 @@ def f(o, p, a): @given(walkable=random_walk()) def test_view(walkable): - ''' + """ Given a walkable location, view that location. - ''' + """ (node, (segments, found)) = walkable assume(found == found) # Hello, nan! We don't want you here. From 4d5fdf201ec760b63fca4ac17440c811044f38a6 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 01:21:20 +0300 Subject: [PATCH 02/37] Add Python >=3.6 constraint for type hints --- setup.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index abce1a3..414702b 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,16 @@ scripts=[], packages=["dpath"], data_files=[], - python_requires=">=3", + + # Type hints are great. + # Function annotations were added in Python 3.0. + # Typing module was added in Python 3.5. + # Variable annotations were added in Python 3.6. + # Python versions that are >=3.6 are more popular. + # (Source: https://github.com/hugovk/pypi-tools/blob/master/README.md) + # + # Conclusion: In order to accommodate type hinting support must be limited to Python versions >=3.6. + python_requires=">=3.6", classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', From e3dc192c6e349b11270e70c318295b390b0b7718 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 01:23:19 +0300 Subject: [PATCH 03/37] Organize imports --- dpath/segments.py | 5 +++-- dpath/util.py | 6 +++--- setup.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 07efd04..5df536e 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,8 +1,9 @@ from copy import deepcopy -from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath import options from fnmatch import fnmatchcase +from dpath import options +from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound + def kvs(node): """ diff --git a/dpath/util.py b/dpath/util.py index 39aaa6e..9f2d9e5 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,8 +1,8 @@ -from collections.abc import MutableMapping -from collections.abc import MutableSequence +from collections.abc import MutableMapping, MutableSequence + +import dpath.segments from dpath import options from dpath.exceptions import InvalidKeyName -import dpath.segments _DEFAULT_SENTINAL = object() MERGE_REPLACE = (1 << 1) diff --git a/setup.py b/setup.py index 414702b..95c7734 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ -from distutils.core import setup -import dpath.version import os +from distutils.core import setup +import dpath.version long_description = open( os.path.join( From 16f511aa4cf673dc7bf8462a998ffd3da226ab8b Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 01:24:27 +0300 Subject: [PATCH 04/37] Fix typo --- dpath/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index 9f2d9e5..ed1bb63 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -4,7 +4,7 @@ from dpath import options from dpath.exceptions import InvalidKeyName -_DEFAULT_SENTINAL = object() +_DEFAULT_SENTINEL = object() MERGE_REPLACE = (1 << 1) MERGE_ADDITIVE = (1 << 2) MERGE_TYPESAFE = (1 << 3) @@ -146,7 +146,7 @@ def f(obj, pair, counter): return changed -def get(obj, glob, separator='/', default=_DEFAULT_SENTINAL): +def get(obj, glob, separator='/', default=_DEFAULT_SENTINEL): """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. @@ -172,7 +172,7 @@ def f(obj, pair, results): results = dpath.segments.fold(obj, f, []) if len(results) == 0: - if default is not _DEFAULT_SENTINAL: + if default is not _DEFAULT_SENTINEL: return default raise KeyError(glob) From 6f8a55828e7e22c72ba4bb9765cfffdb19fdc14d Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 19:57:02 +0300 Subject: [PATCH 05/37] Implement merge types as an enum Additional type hints --- README.rst | 9 ++++----- dpath/util.py | 38 +++++++++++++++++++++++--------------- tests/test_types.py | 2 +- tests/test_util_merge.py | 14 +++++++------- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/README.rst b/README.rst index ea88f5a..cb9ead5 100644 --- a/README.rst +++ b/README.rst @@ -279,14 +279,13 @@ does. Merge source into destination. Like dict.update() but performs deep merging. - flags is an OR'ed combination of MERGE_ADDITIVE, MERGE_REPLACE - MERGE_TYPESAFE. - * MERGE_ADDITIVE : List objects are combined onto one long + flags is an OR'ed combination of MergeType enum members. + * ADDITIVE : List objects are combined onto one long list (NOT a set). This is the default flag. - * MERGE_REPLACE : Instead of combining list objects, when + * REPLACE : Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination with the source. - * MERGE_TYPESAFE : When 2 keys at equal levels are of different + * TYPESAFE : When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source replaces the destination in this situation. diff --git a/dpath/util.py b/dpath/util.py index ed1bb63..e9efb26 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,16 +1,25 @@ from collections.abc import MutableMapping, MutableSequence +from enum import Flag, IntFlag, auto +from typing import Union, List, Any, Dict import dpath.segments from dpath import options from dpath.exceptions import InvalidKeyName _DEFAULT_SENTINEL = object() -MERGE_REPLACE = (1 << 1) -MERGE_ADDITIVE = (1 << 2) -MERGE_TYPESAFE = (1 << 3) -def __safe_path__(path, separator): +class MergeType(IntFlag): + REPLACE = auto() + ADDITIVE = auto() + TYPESAFE = auto() + + +# Type alias for dict path segments where integers are explicitly casted +IntAwareSegment = Union[int, Any] + + +def __safe_path__(path: str, separator: str) -> Union[List[IntAwareSegment], IntAwareSegment]: """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -146,7 +155,7 @@ def f(obj, pair, counter): return changed -def get(obj, glob, separator='/', default=_DEFAULT_SENTINEL): +def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> dict: """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. @@ -156,7 +165,7 @@ def get(obj, glob, separator='/', default=_DEFAULT_SENTINEL): If more than one leaf matches the glob, ValueError is raised. If the glob is not found and a default is not provided, KeyError is raised. """ - if glob == '/': + if glob == "/": return obj globlist = __safe_path__(glob, separator) @@ -233,7 +242,7 @@ def f(obj, pair, result): return dpath.segments.fold(obj, f, {}) -def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): +def merge(dst, src, separator='/', afilter=None, flags=MergeType.ADDITIVE): """ Merge source into destination. Like dict.update() but performs deep merging. @@ -262,14 +271,13 @@ def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): objects that you intend to merge. For further notes see https://github.com/akesterson/dpath-python/issues/58 - flags is an OR'ed combination of MERGE_ADDITIVE, MERGE_REPLACE, - MERGE_TYPESAFE. - * MERGE_ADDITIVE : List objects are combined onto one long + flags is an OR'ed combination of MergeType enum members. + * ADDITIVE : List objects are combined onto one long list (NOT a set). This is the default flag. - * MERGE_REPLACE : Instead of combining list objects, when + * REPLACE : Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination with the source. - * MERGE_TYPESAFE : When 2 keys at equal levels are of different + * TYPESAFE : When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source replaces the destination in this situation. """ @@ -295,7 +303,7 @@ def merger(dst, src, _segments=()): "{}".format(segments)) # Validate src and dst types match. - if flags & MERGE_TYPESAFE: + if flags & MergeType.TYPESAFE: if dpath.segments.has(dst, segments): target = dpath.segments.get(dst, segments) tt = type(target) @@ -332,11 +340,11 @@ def merger(dst, src, _segments=()): # # Pretend we have a sequence and account for the flags. try: - if flags & MERGE_ADDITIVE: + if flags & MergeType.ADDITIVE: target += found continue - if flags & MERGE_REPLACE: + if flags & MergeType.REPLACE: try: target[''] except TypeError: diff --git a/tests/test_types.py b/tests/test_types.py index 82f8c05..39942b9 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -106,7 +106,7 @@ def test_types_merge_simple_list_replace(): "list": TestSequence([0, 1, 2, 3]) }) - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.REPLACE) nose.tools.eq_(dst["list"], TestSequence([7, 8, 9, 10])) diff --git a/tests/test_util_merge.py b/tests/test_util_merge.py index 968a4fa..d5765d2 100644 --- a/tests/test_util_merge.py +++ b/tests/test_util_merge.py @@ -19,7 +19,7 @@ def test_merge_typesafe_and_separator(): } try: - dpath.util.merge(dst, src, flags=(dpath.util.MERGE_ADDITIVE | dpath.util.MERGE_TYPESAFE), separator=";") + dpath.util.merge(dst, src, flags=(dpath.util.MergeType.ADDITIVE | dpath.util.MergeType.TYPESAFE), separator=";") except TypeError as e: assert(str(e).endswith("dict;integer")) @@ -59,7 +59,7 @@ def test_merge_simple_list_additive(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_ADDITIVE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.ADDITIVE) nose.tools.eq_(dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) @@ -71,7 +71,7 @@ def test_merge_simple_list_replace(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.REPLACE) nose.tools.eq_(dst["list"], [7, 8, 9, 10]) @@ -123,7 +123,7 @@ def test_merge_typesafe(): ], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.TYPESAFE) @raises(TypeError) @@ -156,20 +156,20 @@ class tcis(list): assert(dst['ms'][2] == 'c') assert("casserole" in dst["mm"]) - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.TYPESAFE) def test_merge_replace_1(): dct_a = {"a": {"b": [1, 2, 3]}} dct_b = {"a": {"b": [1]}} - dpath.util.merge(dct_a, dct_b, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dct_a, dct_b, flags=dpath.util.MergeType.REPLACE) assert(len(dct_a['a']['b']) == 1) def test_merge_replace_2(): d1 = {'a': [0, 1, 2]} d2 = {'a': ['a']} - dpath.util.merge(d1, d2, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(d1, d2, flags=dpath.util.MergeType.REPLACE) assert(len(d1['a']) == 1) assert(d1['a'][0] == 'a') From fa26b3954e0a649c9ff2b3da497ffa5ff483de99 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 12:24:22 +0300 Subject: [PATCH 06/37] Rename __safe_path__ to _split_path --- dpath/util.py | 12 ++++++------ tests/test_util_paths.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index e9efb26..ab3b2f4 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -19,7 +19,7 @@ class MergeType(IntFlag): IntAwareSegment = Union[int, Any] -def __safe_path__(path: str, separator: str) -> Union[List[IntAwareSegment], IntAwareSegment]: +def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAwareSegment]: """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -67,7 +67,7 @@ def new(obj, path, value, separator='/', creator=None): responsible for creating missing keys at arbitrary levels of the path (see the help for dpath.path.set) """ - segments = __safe_path__(path, separator) + segments = _split_path(path, separator) if creator: return dpath.segments.set(obj, segments, value, creator=creator) return dpath.segments.set(obj, segments, value) @@ -80,7 +80,7 @@ def delete(obj, glob, separator='/', afilter=None): Returns the number of deleted objects. Raises PathNotFound if no paths are found to delete. """ - globlist = __safe_path__(glob, separator) + globlist = _split_path(glob, separator) def f(obj, pair, counter): (segments, value) = pair @@ -135,7 +135,7 @@ def set(obj, glob, value, separator='/', afilter=None): Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. """ - globlist = __safe_path__(glob, separator) + globlist = _split_path(glob, separator) def f(obj, pair, counter): (segments, found) = pair @@ -168,7 +168,7 @@ def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) - if glob == "/": return obj - globlist = __safe_path__(glob, separator) + globlist = _split_path(glob, separator) def f(obj, pair, results): (segments, found) = pair @@ -211,7 +211,7 @@ def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): every element in the document that matched the glob. """ - globlist = __safe_path__(glob, separator) + globlist = _split_path(glob, separator) def keeper(segments, found): """ diff --git a/tests/test_util_paths.py b/tests/test_util_paths.py index 27260fe..6d2c4c9 100644 --- a/tests/test_util_paths.py +++ b/tests/test_util_paths.py @@ -2,7 +2,7 @@ def test_util_safe_path_list(): - res = dpath.util.__safe_path__(["Ignore", "the/separator"], None) + res = dpath.util._split_path(["Ignore", "the/separator"], None) assert(len(res) == 2) assert(res[0] == "Ignore") From d96814be1e92fc6252c33ff4466f7e8f16c3fde2 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 12:43:26 +0300 Subject: [PATCH 07/37] Sort out imports --- dpath/util.py | 113 +++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index ab3b2f4..9a78a1c 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,10 +1,9 @@ from collections.abc import MutableMapping, MutableSequence -from enum import Flag, IntFlag, auto +from enum import IntFlag, auto from typing import Union, List, Any, Dict -import dpath.segments -from dpath import options -from dpath.exceptions import InvalidKeyName +from dpath import options, segments +from dpath.exceptions import InvalidKeyName, PathNotFound _DEFAULT_SENTINEL = object() @@ -29,28 +28,28 @@ def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAw ignored, and is assumed to be part of each key glob. It will not be stripped. """ - if not dpath.segments.leaf(path): - segments = path + if not segments.leaf(path): + split_segments = path else: - segments = path.lstrip(separator).split(separator) + split_segments = path.lstrip(separator).split(separator) # FIXME: This check was in the old internal library, but I can't # see a way it could fail... - for i, segment in enumerate(segments): + for i, segment in enumerate(split_segments): if (separator and (separator in segment)): raise InvalidKeyName("{} at {}[{}] contains the separator '{}'" - "".format(segment, segments, i, separator)) + "".format(segment, split_segments, i, separator)) # Attempt to convert integer segments into actual integers. final = [] - for segment in segments: + for segment in split_segments: try: final.append(int(segment)) except: final.append(segment) - segments = final + split_segments = final - return segments + return split_segments def new(obj, path, value, separator='/', creator=None): @@ -67,10 +66,10 @@ def new(obj, path, value, separator='/', creator=None): responsible for creating missing keys at arbitrary levels of the path (see the help for dpath.path.set) """ - segments = _split_path(path, separator) + split_segments = _split_path(path, separator) if creator: - return dpath.segments.set(obj, segments, value, creator=creator) - return dpath.segments.set(obj, segments, value) + return segments.set(obj, split_segments, value, creator=creator) + return segments.set(obj, split_segments, value) def delete(obj, glob, separator='/', afilter=None): @@ -83,18 +82,18 @@ def delete(obj, glob, separator='/', afilter=None): globlist = _split_path(glob, separator) def f(obj, pair, counter): - (segments, value) = pair + (path_segments, value) = pair # Skip segments if they no longer exist in obj. - if not dpath.segments.has(obj, segments): + if not segments.has(obj, path_segments): return - matched = dpath.segments.match(segments, globlist) - selected = afilter and dpath.segments.leaf(value) and afilter(value) + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(value) and afilter(value) if (matched and not afilter) or selected: - key = segments[-1] - parent = dpath.segments.get(obj, segments[:-1]) + key = path_segments[-1] + parent = segments.get(obj, path_segments[:-1]) try: # Attempt to treat parent like a sequence. @@ -107,7 +106,7 @@ def f(obj, pair, counter): # # Note: In order to achieve proper behavior we are # relying on the reverse iteration of - # non-dictionaries from dpath.segments.kvs(). + # non-dictionaries from segments.kvs(). # Otherwise we'd be unable to delete all the tails # of a list and end up with None values when we # don't need them. @@ -123,9 +122,9 @@ def f(obj, pair, counter): counter[0] += 1 - [deleted] = dpath.segments.foldm(obj, f, [0]) + [deleted] = segments.foldm(obj, f, [0]) if not deleted: - raise dpath.exceptions.PathNotFound("Could not find {0} to delete it".format(glob)) + raise PathNotFound("Could not find {0} to delete it".format(glob)) return deleted @@ -138,20 +137,20 @@ def set(obj, glob, value, separator='/', afilter=None): globlist = _split_path(glob, separator) def f(obj, pair, counter): - (segments, found) = pair + (path_segments, found) = pair # Skip segments if they no longer exist in obj. - if not dpath.segments.has(obj, segments): + if not segments.has(obj, path_segments): return - matched = dpath.segments.match(segments, globlist) - selected = afilter and dpath.segments.leaf(found) and afilter(found) + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(found) and afilter(found) if (matched and not afilter) or (matched and selected): - dpath.segments.set(obj, segments, value, creator=None) + segments.set(obj, path_segments, value, creator=None) counter[0] += 1 - [changed] = dpath.segments.foldm(obj, f, [0]) + [changed] = segments.foldm(obj, f, [0]) return changed @@ -171,14 +170,14 @@ def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) - globlist = _split_path(glob, separator) def f(obj, pair, results): - (segments, found) = pair + (path_segments, found) = pair - if dpath.segments.match(segments, globlist): + if segments.match(path_segments, globlist): results.append(found) if len(results) > 1: return False - results = dpath.segments.fold(obj, f, []) + results = segments.fold(obj, f, []) if len(results) == 0: if default is not _DEFAULT_SENTINEL: @@ -213,33 +212,33 @@ def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): globlist = _split_path(glob, separator) - def keeper(segments, found): + def keeper(path, found): """ Generalized test for use in both yielded and folded cases. Returns True if we want this result. Otherwise returns False. """ - if not dirs and not dpath.segments.leaf(found): + if not dirs and not segments.leaf(found): return False - matched = dpath.segments.match(segments, globlist) + matched = segments.match(path, globlist) selected = afilter and afilter(found) return (matched and not afilter) or (matched and selected) if yielded: def yielder(): - for segments, found in dpath.segments.walk(obj): - if keeper(segments, found): - yield (separator.join(map(dpath.segments.int_str, segments)), found) + for path, found in segments.walk(obj): + if keeper(path, found): + yield (separator.join(map(segments.int_str, path)), found) return yielder() else: def f(obj, pair, result): - (segments, found) = pair + (path, found) = pair - if keeper(segments, found): - dpath.segments.set(result, segments, found, hints=dpath.segments.types(obj, segments)) + if keeper(path, found): + segments.set(result, path, found, hints=segments.types(obj, path)) - return dpath.segments.fold(obj, f, {}) + return segments.fold(obj, f, {}) def merge(dst, src, separator='/', afilter=None, flags=MergeType.ADDITIVE): @@ -293,43 +292,43 @@ def are_both_mutable(o1, o2): return False def merger(dst, src, _segments=()): - for key, found in dpath.segments.kvs(src): + for key, found in segments.kvs(src): # Our current path in the source. - segments = _segments + (key,) + current_path = _segments + (key,) if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(segments)) + "{}".format(current_path)) # Validate src and dst types match. if flags & MergeType.TYPESAFE: - if dpath.segments.has(dst, segments): - target = dpath.segments.get(dst, segments) + if segments.has(dst, current_path): + target = segments.get(dst, current_path) tt = type(target) ft = type(found) if tt != ft: - path = separator.join(segments) + path = separator.join(current_path) raise TypeError("Cannot merge objects of type" "{0} and {1} at {2}" "".format(tt, ft, path)) # Path not present in destination, create it. - if not dpath.segments.has(dst, segments): - dpath.segments.set(dst, segments, found) + if not segments.has(dst, current_path): + segments.set(dst, current_path, found) continue # Retrieve the value in the destination. - target = dpath.segments.get(dst, segments) + target = segments.get(dst, current_path) # If the types don't match, replace it. if ((type(found) != type(target)) and (not are_both_mutable(found, target))): - dpath.segments.set(dst, segments, found) + segments.set(dst, current_path, found) continue # If target is a leaf, the replace it. - if dpath.segments.leaf(target): - dpath.segments.set(dst, segments, found) + if segments.leaf(target): + segments.set(dst, current_path, found) continue # At this point we know: @@ -348,14 +347,14 @@ def merger(dst, src, _segments=()): try: target[''] except TypeError: - dpath.segments.set(dst, segments, found) + segments.set(dst, current_path, found) continue except: raise except: # We have a dictionary like thing and we need to attempt to # recursively merge it. - merger(dst, found, segments) + merger(dst, found, current_path) merger(dst, filtered_src) From 07960492985b7101376101234428a274aa271cb3 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 12:44:10 +0300 Subject: [PATCH 08/37] Remove unnecessary check --- dpath/util.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index 9a78a1c..cd9ed16 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -33,13 +33,6 @@ def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAw else: split_segments = path.lstrip(separator).split(separator) - # FIXME: This check was in the old internal library, but I can't - # see a way it could fail... - for i, segment in enumerate(split_segments): - if (separator and (separator in segment)): - raise InvalidKeyName("{} at {}[{}] contains the separator '{}'" - "".format(segment, split_segments, i, separator)) - # Attempt to convert integer segments into actual integers. final = [] for segment in split_segments: From b6bc7d2d152f5fbfb322e3a0755e75e45f7667cd Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 12:48:44 +0300 Subject: [PATCH 09/37] Rename/remove unused variables --- dpath/segments.py | 7 +++---- dpath/util.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 5df536e..c94507d 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -111,7 +111,7 @@ def expand(segments): expand(keys) -> (..., keys[:-2], keys[:-1]) """ index = 0 - for segment in segments: + for _ in segments: index += 1 yield segments[:index] @@ -185,13 +185,13 @@ def match(segments, glob): path_len = len(segments) glob_len = len(glob) - # Index of the star-star in the glob. - ss = -1 # The star-star normalized glob ('**' has been removed). ss_glob = glob if '**' in glob: + # Index of the star-star in the glob. ss = glob.index('**') + if '**' in glob[ss + 1:]: raise InvalidGlob("Invalid glob. Only one '**' is permitted per glob: {}" "".format(glob)) @@ -362,7 +362,6 @@ def foldm(obj, f, acc): """ pairs = tuple(walk(obj)) for pair in pairs: - (segments, value) = pair if f(obj, pair, acc) is False: break return acc diff --git a/dpath/util.py b/dpath/util.py index cd9ed16..06a12eb 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -162,7 +162,7 @@ def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) - globlist = _split_path(glob, separator) - def f(obj, pair, results): + def f(_, pair, results): (path_segments, found) = pair if segments.match(path_segments, globlist): From 2b73ab3723bcb8b513db898ba9f4dad00629c438 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 13:25:25 +0300 Subject: [PATCH 10/37] Some type hinting --- dpath/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index 06a12eb..f26db63 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -15,10 +15,10 @@ class MergeType(IntFlag): # Type alias for dict path segments where integers are explicitly casted -IntAwareSegment = Union[int, Any] +PathSegment = Union[int, str] -def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAwareSegment]: +def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -45,7 +45,7 @@ def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAw return split_segments -def new(obj, path, value, separator='/', creator=None): +def new(obj: Dict, path: str, value, separator="/", creator=None): """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only From b16c6ad7ade9bb12cccf00055c39f96caeb94e5a Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 13:45:50 +0300 Subject: [PATCH 11/37] More type hinting --- dpath/segments.py | 4 +++- dpath/util.py | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index c94507d..728e041 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,8 +1,10 @@ from copy import deepcopy from fnmatch import fnmatchcase +from typing import List, Sequence, Tuple from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound +from dpath.util import PathSegment def kvs(node): @@ -263,7 +265,7 @@ def extend(thing, index, value=None): return thing -def __default_creator__(current, segments, i, hints=()): +def __default_creator__(current, segments: List[str], i: int, hints: Sequence[Tuple[PathSegment, type]] = ()): """ Create missing path components. If the segment is an int, then it will create a list. Otherwise a dictionary is created. diff --git a/dpath/util.py b/dpath/util.py index f26db63..a0bb851 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,6 +1,6 @@ from collections.abc import MutableMapping, MutableSequence from enum import IntFlag, auto -from typing import Union, List, Any, Dict +from typing import Union, List, Any, Dict, Callable from dpath import options, segments from dpath.exceptions import InvalidKeyName, PathNotFound @@ -17,6 +17,9 @@ class MergeType(IntFlag): # Type alias for dict path segments where integers are explicitly casted PathSegment = Union[int, str] +# Type alias for filter functions +Filter = Callable[[Any], bool] # (Any) -> bool + def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: """ @@ -45,7 +48,8 @@ def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegme return split_segments -def new(obj: Dict, path: str, value, separator="/", creator=None): +# todo: Type hint creator arg +def new(obj: Dict, path: str, value, separator="/", creator=None) -> Dict: """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only @@ -65,7 +69,7 @@ def new(obj: Dict, path: str, value, separator="/", creator=None): return segments.set(obj, split_segments, value) -def delete(obj, glob, separator='/', afilter=None): +def delete(obj: Dict, glob: str, separator='/', afilter: Filter = None) -> int: """ Given a obj, delete all elements that match the glob. @@ -122,7 +126,7 @@ def f(obj, pair, counter): return deleted -def set(obj, glob, value, separator='/', afilter=None): +def set(obj: Dict, glob: str, value, separator='/', afilter: Filter = None) -> int: """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. @@ -147,7 +151,7 @@ def f(obj, pair, counter): return changed -def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> dict: +def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. @@ -183,7 +187,7 @@ def f(_, pair, results): return results[0] -def values(obj, glob, separator='/', afilter=None, dirs=True): +def values(obj: Dict, glob: str, separator='/', afilter: Filter = None, dirs=True): """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). @@ -193,7 +197,7 @@ def values(obj, glob, separator='/', afilter=None, dirs=True): return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] -def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): +def search(obj: Dict, glob: str, yielded=False, separator='/', afilter: Filter = None, dirs=True): """ Given a path glob, return a dictionary containing all keys that matched the given glob. @@ -234,7 +238,7 @@ def f(obj, pair, result): return segments.fold(obj, f, {}) -def merge(dst, src, separator='/', afilter=None, flags=MergeType.ADDITIVE): +def merge(dst: Dict, src: Dict, separator='/', afilter: Filter = None, flags=MergeType.ADDITIVE): """ Merge source into destination. Like dict.update() but performs deep merging. From fdac68fc5bdd3fe0525e12d1bc4d25e5bc859764 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 17 Sep 2021 01:41:13 +0300 Subject: [PATCH 12/37] Remove parens --- dpath/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/util.py b/dpath/util.py index a0bb851..54f532b 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -319,7 +319,7 @@ def merger(dst, src, _segments=()): target = segments.get(dst, current_path) # If the types don't match, replace it. - if ((type(found) != type(target)) and (not are_both_mutable(found, target))): + if type(found) != type(target) and not are_both_mutable(found, target): segments.set(dst, current_path, found) continue From fb77a64be205bf4a67ddb5f7b1644b7ca8ac5868 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sat, 18 Sep 2021 21:20:18 +0300 Subject: [PATCH 13/37] Refactoring and type hinting --- dpath/segments.py | 34 ++++++++++++++++++++-------------- dpath/util.py | 10 ++++++---- tests/test_segments.py | 2 +- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 728e041..3b46af3 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,17 +1,21 @@ from copy import deepcopy from fnmatch import fnmatchcase -from typing import List, Sequence, Tuple +from typing import List, Sequence, Tuple, Iterator, Any, Dict, Union from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound from dpath.util import PathSegment -def kvs(node): +def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: """ - Return a (key, value) iterator for the node. + Returns an iterator which yields tuple pairs of (node index, node value), regardless of node type. - kvs(node) -> (generator -> (key, value)) + * For dict nodes `node.items()` will be returned. + * For sequence nodes (lists/tuples/etc.) a zip between index number and index value will be returned. + * Edge cases will result in an empty iterator being returned. + + make_walkable(node) -> (generator -> (key, value)) """ try: return iter(node.items()) @@ -28,8 +32,6 @@ def kvs(node): def leaf(thing): """ Return True if thing is a leaf, otherwise False. - - leaf(thing) -> bool """ leaves = (bytes, str, int, float, bool, type(None)) @@ -40,8 +42,6 @@ def leafy(thing): """ Same as leaf(thing), but also treats empty sequences and dictionaries as True. - - leafy(thing) -> bool """ try: @@ -59,20 +59,21 @@ def walk(obj, location=()): walk(obj) -> (generator -> (segments, value)) """ if not leaf(obj): - for k, v in kvs(obj): + for k, v in make_walkable(obj): length = None try: length = len(k) - except: + except TypeError: pass if length is not None and length == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " "{}".format(location + (k,))) - yield ((location + (k,)), v) - for k, v in kvs(obj): + yield (location + (k,)), v + + for k, v in make_walkable(obj): for found in walk(v, location + (k,)): yield found @@ -240,7 +241,7 @@ def match(segments, glob): return False -def extend(thing, index, value=None): +def extend(thing: List, index: int, value=None): """ Extend a sequence like thing such that it contains at least index + 1 many elements. The extension values will be None (default). @@ -265,7 +266,12 @@ def extend(thing, index, value=None): return thing -def __default_creator__(current, segments: List[str], i: int, hints: Sequence[Tuple[PathSegment, type]] = ()): +def __default_creator__( + current: Union[Dict, List], + segments: List[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () +): """ Create missing path components. If the segment is an int, then it will create a list. Otherwise a dictionary is created. diff --git a/dpath/util.py b/dpath/util.py index 54f532b..08d7db8 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,6 +1,6 @@ from collections.abc import MutableMapping, MutableSequence from enum import IntFlag, auto -from typing import Union, List, Any, Dict, Callable +from typing import Union, List, Any, Dict, Callable, Sequence, Tuple from dpath import options, segments from dpath.exceptions import InvalidKeyName, PathNotFound @@ -20,6 +20,9 @@ class MergeType(IntFlag): # Type alias for filter functions Filter = Callable[[Any], bool] # (Any) -> bool +# Type alias for creator functions +Creator = Callable[[Union[Dict, List], List[PathSegment], int, Sequence[Tuple[PathSegment, type]]], None] + def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: """ @@ -48,8 +51,7 @@ def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegme return split_segments -# todo: Type hint creator arg -def new(obj: Dict, path: str, value, separator="/", creator=None) -> Dict: +def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only @@ -289,7 +291,7 @@ def are_both_mutable(o1, o2): return False def merger(dst, src, _segments=()): - for key, found in segments.kvs(src): + for key, found in segments.make_walkable(src): # Our current path in the source. current_path = _segments + (key,) diff --git a/tests/test_segments.py b/tests/test_segments.py index 2fdf070..71c11ce 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -43,7 +43,7 @@ def test_kvs(node): Given a node, kvs should produce a key that when used to extract from the node renders the exact same value given. """ - for k, v in api.kvs(node): + for k, v in api.make_walkable(node): assert node[k] is v From 474ba6ddab530aa793d62df486e4d19c4092db24 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 19 Sep 2021 01:09:17 +0300 Subject: [PATCH 14/37] Documentation --- dpath/segments.py | 18 ++++++++++++------ dpath/util.py | 43 ++++++++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 3b46af3..81d218b 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,10 +1,10 @@ from copy import deepcopy from fnmatch import fnmatchcase -from typing import List, Sequence, Tuple, Iterator, Any, Dict, Union +from typing import List, Sequence, Tuple, Iterator, Any, Dict, Union, Optional from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath.util import PathSegment +from dpath.util import PathSegment, Creator, Hints def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: @@ -266,9 +266,9 @@ def extend(thing: List, index: int, value=None): return thing -def __default_creator__( +def _default_creator( current: Union[Dict, List], - segments: List[PathSegment], + segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () ): @@ -301,7 +301,13 @@ def __default_creator__( current[segment] = {} -def set(obj, segments, value, creator=__default_creator__, hints=()): +def set( + obj, + segments: Sequence[PathSegment], + value, + creator: Optional[Creator] = _default_creator, + hints: Hints = () +): """ Set the value in obj at the place indicated by segments. If creator is not None (default __default_creator__), then call the creator function to @@ -323,7 +329,7 @@ def set(obj, segments, value, creator=__default_creator__, hints=()): current[segment] except: if creator is not None: - creator(current, segments, i, hints=hints) + creator(current, segments, i, hints) else: raise diff --git a/dpath/util.py b/dpath/util.py index 08d7db8..6c6f6e8 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,6 +1,6 @@ from collections.abc import MutableMapping, MutableSequence from enum import IntFlag, auto -from typing import Union, List, Any, Dict, Callable, Sequence, Tuple +from typing import Union, List, Any, Dict, Callable, Sequence, Tuple, Optional from dpath import options, segments from dpath.exceptions import InvalidKeyName, PathNotFound @@ -9,19 +9,40 @@ class MergeType(IntFlag): - REPLACE = auto() ADDITIVE = auto() + """List objects are combined onto one long list (NOT a set). This is the default flag.""" + + REPLACE = auto() + """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination + with the source.""" + TYPESAFE = auto() + """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source + replaces the destination in this situation.""" -# Type alias for dict path segments where integers are explicitly casted PathSegment = Union[int, str] +"""Type alias for dict path segments where integers are explicitly casted.""" + +Filter = Callable[[Any], bool] +"""Type alias for filter functions. + +(Any) -> bool""" + +Hints = Sequence[Tuple[PathSegment, type]] +"""Type alias for creator function hint sequences.""" -# Type alias for filter functions -Filter = Callable[[Any], bool] # (Any) -> bool +Creator = Callable[[Union[Dict, List], Sequence[PathSegment], int, Optional[Hints]], None] +"""Type alias for creator functions. -# Type alias for creator functions -Creator = Callable[[Union[Dict, List], List[PathSegment], int, Sequence[Tuple[PathSegment, type]]], None] +Example creator function signature: + + def creator( + current: Union[Dict, List], + segments: Sequence[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () + )""" def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: @@ -270,14 +291,6 @@ def merge(dst: Dict, src: Dict, separator='/', afilter: Filter = None, flags=Mer https://github.com/akesterson/dpath-python/issues/58 flags is an OR'ed combination of MergeType enum members. - * ADDITIVE : List objects are combined onto one long - list (NOT a set). This is the default flag. - * REPLACE : Instead of combining list objects, when - 2 list objects are at an equal depth of merge, replace - the destination with the source. - * TYPESAFE : When 2 keys at equal levels are of different - types, raise a TypeError exception. By default, the source - replaces the destination in this situation. """ filtered_src = search(src, '**', afilter=afilter, separator='/') From 43554c2a30581f6fa7dd5d0434649ae1fd591e6f Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 19 Sep 2021 02:23:36 +0300 Subject: [PATCH 15/37] Use fstrings --- dpath/segments.py | 11 +++++------ dpath/util.py | 24 +++++++++++------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 81d218b..289f3fa 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -70,7 +70,7 @@ def walk(obj, location=()): if length is not None and length == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(location + (k,))) + f"{location + (k,)}") yield (location + (k,)), v for k, v in make_walkable(obj): @@ -87,7 +87,7 @@ def get(obj, segments): current = obj for (i, segment) in enumerate(segments): if leaf(current): - raise PathNotFound('Path: {}[{}]'.format(segments, i)) + raise PathNotFound(f"Path: {segments}[{i}]") current = current[segment] return current @@ -196,8 +196,7 @@ def match(segments, glob): ss = glob.index('**') if '**' in glob[ss + 1:]: - raise InvalidGlob("Invalid glob. Only one '**' is permitted per glob: {}" - "".format(glob)) + raise InvalidGlob(f"Invalid glob. Only one '**' is permitted per glob: {glob}") # Convert '**' segment into multiple '*' segments such that the # lengths of the path and glob match. '**' also can collapse and @@ -249,7 +248,7 @@ def extend(thing: List, index: int, value=None): extend(thing, int) -> [thing..., None, ...] """ try: - expansion = (type(thing)()) + expansion = type(thing)() # Using this rather than the multiply notation in order to support a # wider variety of sequence like things. @@ -335,7 +334,7 @@ def set( current = current[segment] if i != length - 1 and leaf(current): - raise PathNotFound('Path: {}[{}]'.format(segments, i)) + raise PathNotFound(f"Path: {segments}[{i}]") if isinstance(segments[-1], int): extend(current, segments[-1]) diff --git a/dpath/util.py b/dpath/util.py index 6c6f6e8..7c95939 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -92,7 +92,7 @@ def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> return segments.set(obj, split_segments, value) -def delete(obj: Dict, glob: str, separator='/', afilter: Filter = None) -> int: +def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: """ Given a obj, delete all elements that match the glob. @@ -144,12 +144,12 @@ def f(obj, pair, counter): [deleted] = segments.foldm(obj, f, [0]) if not deleted: - raise PathNotFound("Could not find {0} to delete it".format(glob)) + raise PathNotFound(f"Could not find {glob} to delete it") return deleted -def set(obj: Dict, glob: str, value, separator='/', afilter: Filter = None) -> int: +def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. @@ -205,12 +205,12 @@ def f(_, pair, results): raise KeyError(glob) elif len(results) > 1: - raise ValueError("dpath.util.get() globs must match only one leaf : %s" % glob) + raise ValueError(f"dpath.util.get() globs must match only one leaf: {glob}") return results[0] -def values(obj: Dict, glob: str, separator='/', afilter: Filter = None, dirs=True): +def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). @@ -220,7 +220,7 @@ def values(obj: Dict, glob: str, separator='/', afilter: Filter = None, dirs=Tru return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] -def search(obj: Dict, glob: str, yielded=False, separator='/', afilter: Filter = None, dirs=True): +def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): """ Given a path glob, return a dictionary containing all keys that matched the given glob. @@ -249,7 +249,7 @@ def keeper(path, found): def yielder(): for path, found in segments.walk(obj): if keeper(path, found): - yield (separator.join(map(segments.int_str, path)), found) + yield separator.join(map(segments.int_str, path)), found return yielder() else: def f(obj, pair, result): @@ -261,7 +261,7 @@ def f(obj, pair, result): return segments.fold(obj, f, {}) -def merge(dst: Dict, src: Dict, separator='/', afilter: Filter = None, flags=MergeType.ADDITIVE): +def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): """ Merge source into destination. Like dict.update() but performs deep merging. @@ -311,7 +311,7 @@ def merger(dst, src, _segments=()): if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(current_path)) + f"{current_path}") # Validate src and dst types match. if flags & MergeType.TYPESAFE: @@ -321,9 +321,7 @@ def merger(dst, src, _segments=()): ft = type(found) if tt != ft: path = separator.join(current_path) - raise TypeError("Cannot merge objects of type" - "{0} and {1} at {2}" - "".format(tt, ft, path)) + raise TypeError(f"Cannot merge objects of type {tt} and {ft} at {path}") # Path not present in destination, create it. if not segments.has(dst, current_path): @@ -357,7 +355,7 @@ def merger(dst, src, _segments=()): if flags & MergeType.REPLACE: try: - target[''] + target[""] except TypeError: segments.set(dst, current_path, found) continue From 0218ea671467e027a8cf60bca4d886f5dd5ec7e2 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 28 Sep 2021 21:52:34 +0300 Subject: [PATCH 16/37] Remove redundant parentheses --- dpath/segments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 289f3fa..40efb56 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -85,7 +85,7 @@ def get(obj, segments): get(obj, segments) -> value """ current = obj - for (i, segment) in enumerate(segments): + for i, segment in enumerate(segments): if leaf(current): raise PathNotFound(f"Path: {segments}[{i}]") @@ -214,7 +214,7 @@ def match(segments, glob): # If we were successful in matching up the lengths, then we can # compare them using fnmatch. if path_len == len(ss_glob): - for (s, g) in zip(map(int_str, segments), map(int_str, ss_glob)): + for s, g in zip(map(int_str, segments), map(int_str, ss_glob)): # Match the stars we added to the glob to the type of the # segment itself. if g is STAR: From 74fa52deee769435660e1c437143e8d6bab339d8 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 28 Sep 2021 21:52:45 +0300 Subject: [PATCH 17/37] Remove unicode literal --- dpath/segments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/segments.py b/dpath/segments.py index 40efb56..68b6b71 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -221,7 +221,7 @@ def match(segments, glob): if isinstance(s, bytes): g = b'*' else: - g = u'*' + g = '*' # Let's see if the glob matches. We will turn any kind of # exception while attempting to match into a False for the From 7eadc11896fd7a75a74012f8f93cc3e65816d0a0 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 1 Feb 2022 21:52:12 +0200 Subject: [PATCH 18/37] Some type hints and name improvements --- dpath/segments.py | 4 ++-- dpath/util.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 68b6b71..4ee0689 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -140,7 +140,7 @@ def leaves(obj): return filter(lambda p: leafy(p[1]), walk(obj)) -def int_str(segment): +def int_str(segment: PathSegment) -> PathSegment: """ If the segment is an integer, return the string conversion. Otherwise return the segment unchanged. The conversion uses 'str'. @@ -163,7 +163,7 @@ class Star(object): STAR = Star() -def match(segments, glob): +def match(segments: Sequence[PathSegment], glob: Sequence[str]): """ Return True if the segments match the given glob, otherwise False. diff --git a/dpath/util.py b/dpath/util.py index 7c95939..5ba1588 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -230,7 +230,7 @@ def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = every element in the document that matched the glob. """ - globlist = _split_path(glob, separator) + split_glob = _split_path(glob, separator) def keeper(path, found): """ @@ -240,7 +240,7 @@ def keeper(path, found): if not dirs and not segments.leaf(found): return False - matched = segments.match(path, globlist) + matched = segments.match(path, split_glob) selected = afilter and afilter(found) return (matched and not afilter) or (matched and selected) From b2f052e6d296640a763af23ccc6c3efea1b64837 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 2 Feb 2022 22:27:04 +0200 Subject: [PATCH 19/37] Small refactor in deletion code --- dpath/util.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index d6622a8..5350a75 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -116,9 +116,13 @@ def f(obj, pair, counter): key = path_segments[-1] parent = segments.get(obj, path_segments[:-1]) - try: - # Attempt to treat parent like a sequence. - parent[0] + # Deletion behavior depends on parent type + if isinstance(parent, dict): + del parent[key] + + else: + # Handle sequence types + # TODO: Consider cases where type isn't a simple list (e.g. set) if len(parent) - 1 == key: # Removing the last element of a sequence. It can be @@ -132,14 +136,12 @@ def f(obj, pair, counter): # of a list and end up with None values when we # don't need them. del parent[key] + else: # This key can't be removed completely because it # would affect the order of items that remain in our # result. parent[key] = None - except: - # Attempt to treat parent like a dictionary instead. - del parent[key] counter[0] += 1 From f9c142bdbfeb5677ef1e14a316489615b5058741 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 2 Feb 2022 22:35:39 +0200 Subject: [PATCH 20/37] Move custom type definitions to dedicated file --- dpath/segments.py | 2 +- dpath/types.py | 39 +++++++++++++++++++++++++++++++++++++++ dpath/util.py | 41 ++--------------------------------------- 3 files changed, 42 insertions(+), 40 deletions(-) create mode 100644 dpath/types.py diff --git a/dpath/segments.py b/dpath/segments.py index 4ee0689..7c5816c 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -4,7 +4,7 @@ from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath.util import PathSegment, Creator, Hints +from dpath.types import PathSegment, Creator, Hints def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: diff --git a/dpath/types.py b/dpath/types.py new file mode 100644 index 0000000..2b44ce6 --- /dev/null +++ b/dpath/types.py @@ -0,0 +1,39 @@ +from enum import IntFlag, auto +from typing import Union, Any, Callable, Sequence, Tuple, Dict, List, Optional + + +class MergeType(IntFlag): + ADDITIVE = auto() + """List objects are combined onto one long list (NOT a set). This is the default flag.""" + + REPLACE = auto() + """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination + with the source.""" + + TYPESAFE = auto() + """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source + replaces the destination in this situation.""" + + +PathSegment = Union[int, str] +"""Type alias for dict path segments where integers are explicitly casted.""" + +Filter = Callable[[Any], bool] +"""Type alias for filter functions. + +(Any) -> bool""" + +Hints = Sequence[Tuple[PathSegment, type]] +"""Type alias for creator function hint sequences.""" + +Creator = Callable[[Union[Dict, List], Sequence[PathSegment], int, Optional[Hints]], None] +"""Type alias for creator functions. + +Example creator function signature: + + def creator( + current: Union[Dict, List], + segments: Sequence[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () + )""" \ No newline at end of file diff --git a/dpath/util.py b/dpath/util.py index 5350a75..2384578 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,50 +1,13 @@ from collections.abc import MutableMapping, MutableSequence -from enum import IntFlag, auto -from typing import Union, List, Any, Dict, Callable, Sequence, Tuple, Optional +from typing import Union, List, Any, Dict from dpath import options, segments from dpath.exceptions import InvalidKeyName, PathNotFound +from dpath.types import PathSegment, Filter, Creator, MergeType _DEFAULT_SENTINEL = object() -class MergeType(IntFlag): - ADDITIVE = auto() - """List objects are combined onto one long list (NOT a set). This is the default flag.""" - - REPLACE = auto() - """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination - with the source.""" - - TYPESAFE = auto() - """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source - replaces the destination in this situation.""" - - -PathSegment = Union[int, str] -"""Type alias for dict path segments where integers are explicitly casted.""" - -Filter = Callable[[Any], bool] -"""Type alias for filter functions. - -(Any) -> bool""" - -Hints = Sequence[Tuple[PathSegment, type]] -"""Type alias for creator function hint sequences.""" - -Creator = Callable[[Union[Dict, List], Sequence[PathSegment], int, Optional[Hints]], None] -"""Type alias for creator functions. - -Example creator function signature: - - def creator( - current: Union[Dict, List], - segments: Sequence[PathSegment], - i: int, - hints: Sequence[Tuple[PathSegment, type]] = () - )""" - - def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: """ Given a path and separator, return a tuple of segments. If path is From 86fce6160ad87a3ea5b693cb7e376ca0f93439f0 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 2 Feb 2022 22:35:59 +0200 Subject: [PATCH 21/37] Use MutableMapping instead of dict --- dpath/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/util.py b/dpath/util.py index 2384578..5200f81 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -80,7 +80,7 @@ def f(obj, pair, counter): parent = segments.get(obj, path_segments[:-1]) # Deletion behavior depends on parent type - if isinstance(parent, dict): + if isinstance(parent, MutableMapping): del parent[key] else: From a00adc3ce59197308f8114ce0f6b3b549704635d Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 00:32:51 +0200 Subject: [PATCH 22/37] Catch specific exceptions --- dpath/segments.py | 2 +- dpath/util.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 7c5816c..f452d12 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -326,7 +326,7 @@ def set( # Unfortunately, for our use, 'x in thing' for lists checks # values, not keys whereas dicts check keys. current[segment] - except: + except (KeyError, IndexError): if creator is not None: creator(current, segments, i, hints) else: diff --git a/dpath/util.py b/dpath/util.py index 5200f81..a3c5450 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -29,7 +29,7 @@ def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegme for segment in split_segments: try: final.append(int(segment)) - except: + except ValueError: final.append(segment) split_segments = final @@ -325,9 +325,9 @@ def merger(dst, src, _segments=()): except TypeError: segments.set(dst, current_path, found) continue - except: + except Exception: raise - except: + except Exception: # We have a dictionary like thing and we need to attempt to # recursively merge it. merger(dst, found, current_path) From 8d3a36f49ab62820488edc089decfe3fb693f5e8 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 00:38:16 +0200 Subject: [PATCH 23/37] Add exports --- dpath/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dpath/__init__.py b/dpath/__init__.py index e69de29..922e139 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -0,0 +1,16 @@ +from dpath.util import new, delete, set, get, values, search, merge + +__all__ = [ + "new", + "delete", + "set", + "get", + "values", + "search", + "merge", + "exceptions", + "options", + "segments", + "types", + "version", +] From 3198c54f19d9b04805cd425ef264d4ffc905fbef Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 01:00:07 +0200 Subject: [PATCH 24/37] Move utils functions to top level --- README.rst | 58 ++++---- dpath/__init__.py | 341 +++++++++++++++++++++++++++++++++++++++++++++- dpath/util.py | 337 ++++----------------------------------------- 3 files changed, 394 insertions(+), 342 deletions(-) diff --git a/README.rst b/README.rst index ccb6608..f88f1ed 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ Using Dpath .. code-block:: python - import dpath.util + import dpath Separators ========== @@ -62,8 +62,8 @@ key '43' in the 'b' hash which is in the 'a' hash". That's easy. .. code-block:: pycon - >>> help(dpath.util.get) - Help on function get in module dpath.util: + >>> help(dpath.get) + Help on function get in module dpath: get(obj, glob, separator='/') Given an object which contains only one possible match for the given glob, @@ -72,7 +72,7 @@ key '43' in the 'b' hash which is in the 'a' hash". That's easy. If more than one leaf matches the glob, ValueError is raised. If the glob is not found, KeyError is raised. - >>> dpath.util.get(x, '/a/b/43') + >>> dpath.get(x, '/a/b/43') 30 Or you could say "Give me a new dictionary with the values of all @@ -80,8 +80,8 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. .. code-block:: pycon - >>> help(dpath.util.search) - Help on function search in module dpath.util: + >>> help(dpath.search) + Help on function search in module dpath: search(obj, glob, yielded=False) Given a path glob, return a dictionary containing all keys @@ -95,7 +95,7 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. .. code-block:: pycon - >>> result = dpath.util.search(x, "a/b/[cd]") + >>> result = dpath.search(x, "a/b/[cd]") >>> print(json.dumps(result, indent=4, sort_keys=True)) { "a": { @@ -115,7 +115,7 @@ not get a merged view? .. code-block:: pycon - >>> for x in dpath.util.search(x, "a/b/[cd]", yielded=True): print(x) + >>> for x in dpath.search(x, "a/b/[cd]", yielded=True): print(x) ... ('a/b/c', []) ('a/b/d', ['red', 'buggy', 'bumpers']) @@ -125,8 +125,8 @@ don't care about the paths they were found at: .. code-block:: pycon - >>> help(dpath.util.values) - Help on function values in module dpath.util: + >>> help(dpath.values) + Help on function values in module dpath: values(obj, glob, separator='/', afilter=None, dirs=True) Given an object and a path glob, return an array of all values which match @@ -134,7 +134,7 @@ don't care about the paths they were found at: and it is primarily a shorthand for a list comprehension over a yielded search call. - >>> dpath.util.values(x, '/a/b/d/*') + >>> dpath.values(x, '/a/b/d/*') ['red', 'buggy', 'bumpers'] Example: Setting existing keys @@ -145,14 +145,14 @@ value 'Waffles'. .. code-block:: pycon - >>> help(dpath.util.set) - Help on function set in module dpath.util: + >>> help(dpath.set) + Help on function set in module dpath: set(obj, glob, value) Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. - >>> dpath.util.set(x, 'a/b/[cd]', 'Waffles') + >>> dpath.set(x, 'a/b/[cd]', 'Waffles') 2 >>> print(json.dumps(x, indent=4, sort_keys=True)) { @@ -175,8 +175,8 @@ necessary to get to the terminus. .. code-block:: pycon - >>> help(dpath.util.new) - Help on function new in module dpath.util: + >>> help(dpath.new) + Help on function new in module dpath: new(obj, path, value) Set the element at the terminus of path to value, and create @@ -187,7 +187,7 @@ necessary to get to the terminus. characters in it, they will become part of the resulting keys - >>> dpath.util.new(x, 'a/b/e/f/g', "Roffle") + >>> dpath.new(x, 'a/b/e/f/g', "Roffle") >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { @@ -211,8 +211,8 @@ object with None entries in order to make it big enough: .. code-block:: pycon - >>> dpath.util.new(x, 'a/b/e/f/h', []) - >>> dpath.util.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') + >>> dpath.new(x, 'a/b/e/f/h', []) + >>> dpath.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { @@ -251,11 +251,11 @@ Handy! Example: Deleting Existing Keys =============================== -To delete keys in an object, use dpath.util.delete, which accepts the same globbing syntax as the other methods. +To delete keys in an object, use dpath.delete, which accepts the same globbing syntax as the other methods. .. code-block:: pycon - >>> help(dpath.util.delete) + >>> help(dpath.delete) delete(obj, glob, separator='/', afilter=None): Given a path glob, delete all elements that match the glob. @@ -266,14 +266,14 @@ To delete keys in an object, use dpath.util.delete, which accepts the same globb Example: Merging ================ -Also, check out dpath.util.merge. The python dict update() method is +Also, check out dpath.merge. The python dict update() method is great and all but doesn't handle merging dictionaries deeply. This one does. .. code-block:: pycon - >>> help(dpath.util.merge) - Help on function merge in module dpath.util: + >>> help(dpath.merge) + Help on function merge in module dpath: merge(dst, src, afilter=None, flags=4, _path='') Merge source into destination. Like dict.update() but performs @@ -310,7 +310,7 @@ does. "c": "RoffleWaffles" } } - >>> dpath.util.merge(x, y) + >>> dpath.merge(x, y) >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { @@ -390,7 +390,7 @@ them: ... return True ... return False ... - >>> result = dpath.util.search(x, '**', afilter=afilter) + >>> result = dpath.search(x, '**', afilter=afilter) >>> print(json.dumps(result, indent=4, sort_keys=True)) { "a": { @@ -429,18 +429,18 @@ Separator got you down? Use lists as paths The default behavior in dpath is to assume that the path given is a string, which must be tokenized by splitting at the separator to yield a distinct set of path components against which dictionary keys can be individually glob tested. However, this presents a problem when you want to use paths that have a separator in their name; the tokenizer cannot properly understand what you mean by '/a/b/c' if it is possible for '/' to exist as a valid character in a key name. -To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.util.* support the use of a list instead of a string as a path. So for example: +To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.* support the use of a list instead of a string as a path. So for example: .. code-block:: python >>> x = { 'a': {'b/c': 0}} - >>> dpath.util.get(['a', 'b/c']) + >>> dpath.get(['a', 'b/c']) 0 dpath.segments : The Low-Level Backend ====================================== -dpath.util is where you want to spend your time: this library has the friendly +dpath is where you want to spend your time: this library has the friendly functions that will understand simple string globs, afilter functions, etc. dpath.segments is the backend pathing library. It passes around tuples of path diff --git a/dpath/__init__.py b/dpath/__init__.py index 922e139..a3180bd 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -1,5 +1,3 @@ -from dpath.util import new, delete, set, get, values, search, merge - __all__ = [ "new", "delete", @@ -14,3 +12,342 @@ "types", "version", ] + +from collections.abc import MutableMapping, MutableSequence +from typing import Union, List, Dict, Any + +from dpath import segments, options +from dpath.exceptions import InvalidKeyName, PathNotFound +from dpath.types import MergeType, PathSegment, Creator, Filter + +_DEFAULT_SENTINEL = object() + + +def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: + """ + Given a path and separator, return a tuple of segments. If path is + already a non-leaf thing, return it. + + Note that a string path with the separator at index[0] will have the + separator stripped off. If you pass a list path, the separator is + ignored, and is assumed to be part of each key glob. It will not be + stripped. + """ + if not segments.leaf(path): + split_segments = path + else: + split_segments = path.lstrip(separator).split(separator) + + if options.CONVERT_INT_LIKE_SEGMENTS: + # Attempt to convert integer segments into actual integers. + final = [] + for segment in split_segments: + try: + final.append(int(segment)) + except ValueError: + final.append(segment) + split_segments = final + + return split_segments + + +def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: + """ + Set the element at the terminus of path to value, and create + it if it does not exist (as opposed to 'set' that can only + change existing keys). + + path will NOT be treated like a glob. If it has globbing + characters in it, they will become part of the resulting + keys + + creator allows you to pass in a creator method that is + responsible for creating missing keys at arbitrary levels of + the path (see the help for dpath.path.set) + """ + split_segments = _split_path(path, separator) + if creator: + return segments.set(obj, split_segments, value, creator=creator) + return segments.set(obj, split_segments, value) + + +def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: + """ + Given a obj, delete all elements that match the glob. + + Returns the number of deleted objects. Raises PathNotFound if no paths are + found to delete. + """ + globlist = _split_path(glob, separator) + + def f(obj, pair, counter): + (path_segments, value) = pair + + # Skip segments if they no longer exist in obj. + if not segments.has(obj, path_segments): + return + + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(value) and afilter(value) + + if (matched and not afilter) or selected: + key = path_segments[-1] + parent = segments.get(obj, path_segments[:-1]) + + # Deletion behavior depends on parent type + if isinstance(parent, MutableMapping): + del parent[key] + + else: + # Handle sequence types + # TODO: Consider cases where type isn't a simple list (e.g. set) + + if len(parent) - 1 == key: + # Removing the last element of a sequence. It can be + # truly removed without affecting the ordering of + # remaining items. + # + # Note: In order to achieve proper behavior we are + # relying on the reverse iteration of + # non-dictionaries from segments.kvs(). + # Otherwise we'd be unable to delete all the tails + # of a list and end up with None values when we + # don't need them. + del parent[key] + + else: + # This key can't be removed completely because it + # would affect the order of items that remain in our + # result. + parent[key] = None + + counter[0] += 1 + + [deleted] = segments.foldm(obj, f, [0]) + if not deleted: + raise PathNotFound(f"Could not find {glob} to delete it") + + return deleted + + +def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: + """ + Given a path glob, set all existing elements in the document + to the given value. Returns the number of elements changed. + """ + globlist = _split_path(glob, separator) + + def f(obj, pair, counter): + (path_segments, found) = pair + + # Skip segments if they no longer exist in obj. + if not segments.has(obj, path_segments): + return + + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(found) and afilter(found) + + if (matched and not afilter) or (matched and selected): + segments.set(obj, path_segments, value, creator=None) + counter[0] += 1 + + [changed] = segments.foldm(obj, f, [0]) + return changed + + +def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: + """ + Given an object which contains only one possible match for the given glob, + return the value for the leaf matching the given glob. + If the glob is not found and a default is provided, + the default is returned. + + If more than one leaf matches the glob, ValueError is raised. If the glob is + not found and a default is not provided, KeyError is raised. + """ + if glob == "/": + return obj + + globlist = _split_path(glob, separator) + + def f(_, pair, results): + (path_segments, found) = pair + + if segments.match(path_segments, globlist): + results.append(found) + if len(results) > 1: + return False + + results = segments.fold(obj, f, []) + + if len(results) == 0: + if default is not _DEFAULT_SENTINEL: + return default + + raise KeyError(glob) + elif len(results) > 1: + raise ValueError(f"dpath.util.get() globs must match only one leaf: {glob}") + + return results[0] + + +def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): + """ + Given an object and a path glob, return an array of all values which match + the glob. The arguments to this function are identical to those of search(). + """ + yielded = True + + return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] + + +def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): + """ + Given a path glob, return a dictionary containing all keys + that matched the given glob. + + If 'yielded' is true, then a dictionary will not be returned. + Instead tuples will be yielded in the form of (path, value) for + every element in the document that matched the glob. + """ + + split_glob = _split_path(glob, separator) + + def keeper(path, found): + """ + Generalized test for use in both yielded and folded cases. + Returns True if we want this result. Otherwise returns False. + """ + if not dirs and not segments.leaf(found): + return False + + matched = segments.match(path, split_glob) + selected = afilter and afilter(found) + + return (matched and not afilter) or (matched and selected) + + if yielded: + def yielder(): + for path, found in segments.walk(obj): + if keeper(path, found): + yield separator.join(map(segments.int_str, path)), found + + return yielder() + else: + def f(obj, pair, result): + (path, found) = pair + + if keeper(path, found): + segments.set(result, path, found, hints=segments.types(obj, path)) + + return segments.fold(obj, f, {}) + + +def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): + """ + Merge source into destination. Like dict.update() but performs deep + merging. + + NOTE: This does not do a deep copy of the source object. Applying merge + will result in references to src being present in the dst tree. If you do + not want src to potentially be modified by other changes in dst (e.g. more + merge calls), then use a deep copy of src. + + NOTE that merge() does NOT copy objects - it REFERENCES. If you merge + take these two dictionaries: + + >>> a = {'a': [0] } + >>> b = {'a': [1] } + + ... and you merge them into an empty dictionary, like so: + + >>> d = {} + >>> dpath.merge(d, a) + >>> dpath.merge(d, b) + + ... you might be surprised to find that a['a'] now contains [0, 1]. + This is because merge() says (d['a'] = a['a']), and thus creates a reference. + This reference is then modified when b is merged, causing both d and + a to have ['a'][0, 1]. To avoid this, make your own deep copies of source + objects that you intend to merge. For further notes see + https://github.com/akesterson/dpath-python/issues/58 + + flags is an OR'ed combination of MergeType enum members. + """ + filtered_src = search(src, '**', afilter=afilter, separator='/') + + def are_both_mutable(o1, o2): + mapP = isinstance(o1, MutableMapping) and isinstance(o2, MutableMapping) + seqP = isinstance(o1, MutableSequence) and isinstance(o2, MutableSequence) + + if mapP or seqP: + return True + + return False + + def merger(dst, src, _segments=()): + for key, found in segments.make_walkable(src): + # Our current path in the source. + current_path = _segments + (key,) + + if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: + raise InvalidKeyName("Empty string keys not allowed without " + "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " + f"{current_path}") + + # Validate src and dst types match. + if flags & MergeType.TYPESAFE: + if segments.has(dst, current_path): + target = segments.get(dst, current_path) + tt = type(target) + ft = type(found) + if tt != ft: + path = separator.join(current_path) + raise TypeError(f"Cannot merge objects of type {tt} and {ft} at {path}") + + # Path not present in destination, create it. + if not segments.has(dst, current_path): + segments.set(dst, current_path, found) + continue + + # Retrieve the value in the destination. + target = segments.get(dst, current_path) + + # If the types don't match, replace it. + if type(found) != type(target) and not are_both_mutable(found, target): + segments.set(dst, current_path, found) + continue + + # If target is a leaf, the replace it. + if segments.leaf(target): + segments.set(dst, current_path, found) + continue + + # At this point we know: + # + # * The target exists. + # * The types match. + # * The target isn't a leaf. + # + # Pretend we have a sequence and account for the flags. + try: + if flags & MergeType.ADDITIVE: + target += found + continue + + if flags & MergeType.REPLACE: + try: + target[""] + except TypeError: + segments.set(dst, current_path, found) + continue + except Exception: + raise + except Exception: + # We have a dictionary like thing and we need to attempt to + # recursively merge it. + merger(dst, found, current_path) + + merger(dst, filtered_src) + + return dst diff --git a/dpath/util.py b/dpath/util.py index a3c5450..c3e99d1 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,337 +1,52 @@ -from collections.abc import MutableMapping, MutableSequence -from typing import Union, List, Any, Dict +import warnings +from typing import Any, Dict -from dpath import options, segments -from dpath.exceptions import InvalidKeyName, PathNotFound -from dpath.types import PathSegment, Filter, Creator, MergeType +import dpath +from dpath import _DEFAULT_SENTINEL +from dpath.types import Filter, Creator, MergeType -_DEFAULT_SENTINEL = object() +def deprecated(func): + message =\ + "The dpath.util package is being deprecated. All util functions have been moved to dpath package top level." -def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: - """ - Given a path and separator, return a tuple of segments. If path is - already a non-leaf thing, return it. + def wrapper(*args, **kwargs): + warnings.warn(message, DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) - Note that a string path with the separator at index[0] will have the - separator stripped off. If you pass a list path, the separator is - ignored, and is assumed to be part of each key glob. It will not be - stripped. - """ - if not segments.leaf(path): - split_segments = path - else: - split_segments = path.lstrip(separator).split(separator) - - if options.CONVERT_INT_LIKE_SEGMENTS: - # Attempt to convert integer segments into actual integers. - final = [] - for segment in split_segments: - try: - final.append(int(segment)) - except ValueError: - final.append(segment) - split_segments = final - - return split_segments + return wrapper +@deprecated def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: - """ - Set the element at the terminus of path to value, and create - it if it does not exist (as opposed to 'set' that can only - change existing keys). - - path will NOT be treated like a glob. If it has globbing - characters in it, they will become part of the resulting - keys - - creator allows you to pass in a creator method that is - responsible for creating missing keys at arbitrary levels of - the path (see the help for dpath.path.set) - """ - split_segments = _split_path(path, separator) - if creator: - return segments.set(obj, split_segments, value, creator=creator) - return segments.set(obj, split_segments, value) + return dpath.new(obj, path, value, separator, creator) +@deprecated def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: - """ - Given a obj, delete all elements that match the glob. - - Returns the number of deleted objects. Raises PathNotFound if no paths are - found to delete. - """ - globlist = _split_path(glob, separator) - - def f(obj, pair, counter): - (path_segments, value) = pair - - # Skip segments if they no longer exist in obj. - if not segments.has(obj, path_segments): - return - - matched = segments.match(path_segments, globlist) - selected = afilter and segments.leaf(value) and afilter(value) - - if (matched and not afilter) or selected: - key = path_segments[-1] - parent = segments.get(obj, path_segments[:-1]) - - # Deletion behavior depends on parent type - if isinstance(parent, MutableMapping): - del parent[key] - - else: - # Handle sequence types - # TODO: Consider cases where type isn't a simple list (e.g. set) - - if len(parent) - 1 == key: - # Removing the last element of a sequence. It can be - # truly removed without affecting the ordering of - # remaining items. - # - # Note: In order to achieve proper behavior we are - # relying on the reverse iteration of - # non-dictionaries from segments.kvs(). - # Otherwise we'd be unable to delete all the tails - # of a list and end up with None values when we - # don't need them. - del parent[key] - - else: - # This key can't be removed completely because it - # would affect the order of items that remain in our - # result. - parent[key] = None - - counter[0] += 1 - - [deleted] = segments.foldm(obj, f, [0]) - if not deleted: - raise PathNotFound(f"Could not find {glob} to delete it") - - return deleted + return dpath.delete(obj, glob, separator, afilter) +@deprecated def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: - """ - Given a path glob, set all existing elements in the document - to the given value. Returns the number of elements changed. - """ - globlist = _split_path(glob, separator) - - def f(obj, pair, counter): - (path_segments, found) = pair - - # Skip segments if they no longer exist in obj. - if not segments.has(obj, path_segments): - return - - matched = segments.match(path_segments, globlist) - selected = afilter and segments.leaf(found) and afilter(found) - - if (matched and not afilter) or (matched and selected): - segments.set(obj, path_segments, value, creator=None) - counter[0] += 1 - - [changed] = segments.foldm(obj, f, [0]) - return changed + return dpath.set(obj, glob, value, separator, afilter) +@deprecated def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: - """ - Given an object which contains only one possible match for the given glob, - return the value for the leaf matching the given glob. - If the glob is not found and a default is provided, - the default is returned. - - If more than one leaf matches the glob, ValueError is raised. If the glob is - not found and a default is not provided, KeyError is raised. - """ - if glob == "/": - return obj - - globlist = _split_path(glob, separator) - - def f(_, pair, results): - (path_segments, found) = pair - - if segments.match(path_segments, globlist): - results.append(found) - if len(results) > 1: - return False - - results = segments.fold(obj, f, []) - - if len(results) == 0: - if default is not _DEFAULT_SENTINEL: - return default - - raise KeyError(glob) - elif len(results) > 1: - raise ValueError(f"dpath.util.get() globs must match only one leaf: {glob}") - - return results[0] + return dpath.get(obj, glob, separator, default) +@deprecated def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): - """ - Given an object and a path glob, return an array of all values which match - the glob. The arguments to this function are identical to those of search(). - """ - yielded = True - - return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] + return dpath.values(obj, glob, separator, afilter, dirs) +@deprecated def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): - """ - Given a path glob, return a dictionary containing all keys - that matched the given glob. - - If 'yielded' is true, then a dictionary will not be returned. - Instead tuples will be yielded in the form of (path, value) for - every element in the document that matched the glob. - """ - - split_glob = _split_path(glob, separator) - - def keeper(path, found): - """ - Generalized test for use in both yielded and folded cases. - Returns True if we want this result. Otherwise returns False. - """ - if not dirs and not segments.leaf(found): - return False - - matched = segments.match(path, split_glob) - selected = afilter and afilter(found) - - return (matched and not afilter) or (matched and selected) - - if yielded: - def yielder(): - for path, found in segments.walk(obj): - if keeper(path, found): - yield separator.join(map(segments.int_str, path)), found - return yielder() - else: - def f(obj, pair, result): - (path, found) = pair - - if keeper(path, found): - segments.set(result, path, found, hints=segments.types(obj, path)) - - return segments.fold(obj, f, {}) + return dpath.search(obj, glob, yielded, separator, afilter, dirs) +@deprecated def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): - """ - Merge source into destination. Like dict.update() but performs deep - merging. - - NOTE: This does not do a deep copy of the source object. Applying merge - will result in references to src being present in the dst tree. If you do - not want src to potentially be modified by other changes in dst (e.g. more - merge calls), then use a deep copy of src. - - NOTE that merge() does NOT copy objects - it REFERENCES. If you merge - take these two dictionaries: - - >>> a = {'a': [0] } - >>> b = {'a': [1] } - - ... and you merge them into an empty dictionary, like so: - - >>> d = {} - >>> dpath.util.merge(d, a) - >>> dpath.util.merge(d, b) - - ... you might be surprised to find that a['a'] now contains [0, 1]. - This is because merge() says (d['a'] = a['a']), and thus creates a reference. - This reference is then modified when b is merged, causing both d and - a to have ['a'][0, 1]. To avoid this, make your own deep copies of source - objects that you intend to merge. For further notes see - https://github.com/akesterson/dpath-python/issues/58 - - flags is an OR'ed combination of MergeType enum members. - """ - filtered_src = search(src, '**', afilter=afilter, separator='/') - - def are_both_mutable(o1, o2): - mapP = isinstance(o1, MutableMapping) and isinstance(o2, MutableMapping) - seqP = isinstance(o1, MutableSequence) and isinstance(o2, MutableSequence) - - if mapP or seqP: - return True - - return False - - def merger(dst, src, _segments=()): - for key, found in segments.make_walkable(src): - # Our current path in the source. - current_path = _segments + (key,) - - if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: - raise InvalidKeyName("Empty string keys not allowed without " - "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - f"{current_path}") - - # Validate src and dst types match. - if flags & MergeType.TYPESAFE: - if segments.has(dst, current_path): - target = segments.get(dst, current_path) - tt = type(target) - ft = type(found) - if tt != ft: - path = separator.join(current_path) - raise TypeError(f"Cannot merge objects of type {tt} and {ft} at {path}") - - # Path not present in destination, create it. - if not segments.has(dst, current_path): - segments.set(dst, current_path, found) - continue - - # Retrieve the value in the destination. - target = segments.get(dst, current_path) - - # If the types don't match, replace it. - if type(found) != type(target) and not are_both_mutable(found, target): - segments.set(dst, current_path, found) - continue - - # If target is a leaf, the replace it. - if segments.leaf(target): - segments.set(dst, current_path, found) - continue - - # At this point we know: - # - # * The target exists. - # * The types match. - # * The target isn't a leaf. - # - # Pretend we have a sequence and account for the flags. - try: - if flags & MergeType.ADDITIVE: - target += found - continue - - if flags & MergeType.REPLACE: - try: - target[""] - except TypeError: - segments.set(dst, current_path, found) - continue - except Exception: - raise - except Exception: - # We have a dictionary like thing and we need to attempt to - # recursively merge it. - merger(dst, found, current_path) - - merger(dst, filtered_src) - - return dst + return dpath.merge(dst, src, separator, afilter, flags), From 605a7567e385ffe4f261c284e71623fa91f58035 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 01:21:27 +0200 Subject: [PATCH 25/37] Fix tests --- tests/test_segments.py | 2 +- tests/test_types.py | 3 ++- tests/test_util_merge.py | 13 +++++++------ tests/test_util_paths.py | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/test_segments.py b/tests/test_segments.py index 40a9f9d..fb6a8bc 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -170,7 +170,7 @@ def test_kvs(self, node): Given a node, kvs should produce a key that when used to extract from the node renders the exact same value given. ''' - for k, v in api.kvs(node): + for k, v in api.make_walkable(node): assert node[k] is v @given(random_leaf) diff --git a/tests/test_types.py b/tests/test_types.py index 46b12b6..56eb98f 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -3,6 +3,7 @@ from nose2.tools.such import helper import dpath.util +from dpath import MergeType class TestMapping(MutableMapping): @@ -99,7 +100,7 @@ def test_types_merge_simple_list_replace(): "list": TestSequence([0, 1, 2, 3]) }) - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dst, src, flags=MergeType.REPLACE) assert dst["list"] == TestSequence([7, 8, 9, 10]), "%r != %r" % (dst["list"], TestSequence([7, 8, 9, 10])) diff --git a/tests/test_util_merge.py b/tests/test_util_merge.py index 9ea9e84..695e58f 100644 --- a/tests/test_util_merge.py +++ b/tests/test_util_merge.py @@ -4,6 +4,7 @@ import dpath.util +from dpath import MergeType def test_merge_typesafe_and_separator(): @@ -59,7 +60,7 @@ def test_merge_simple_list_additive(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_ADDITIVE) + dpath.util.merge(dst, src, flags=MergeType.ADDITIVE) assert dst["list"] == [0, 1, 2, 3, 7, 8, 9, 10], "%r != %r" % (dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) @@ -71,7 +72,7 @@ def test_merge_simple_list_replace(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.REPLACE) assert dst["list"] == [7, 8, 9, 10], "%r != %r" % (dst["list"], [7, 8, 9, 10]) @@ -122,7 +123,7 @@ def test_merge_typesafe(): ], } - helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MergeType.TYPESAFE) def test_merge_mutables(): @@ -154,20 +155,20 @@ class tcis(list): assert dst['ms'][2] == 'c' assert "casserole" in dst["mm"] - helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MergeType.TYPESAFE) def test_merge_replace_1(): dct_a = {"a": {"b": [1, 2, 3]}} dct_b = {"a": {"b": [1]}} - dpath.util.merge(dct_a, dct_b, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dct_a, dct_b, flags=dpath.util.MergeType.REPLACE) assert len(dct_a['a']['b']) == 1 def test_merge_replace_2(): d1 = {'a': [0, 1, 2]} d2 = {'a': ['a']} - dpath.util.merge(d1, d2, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(d1, d2, flags=dpath.util.MergeType.REPLACE) assert len(d1['a']) == 1 assert d1['a'][0] == 'a' diff --git a/tests/test_util_paths.py b/tests/test_util_paths.py index f84e43f..49bec1b 100644 --- a/tests/test_util_paths.py +++ b/tests/test_util_paths.py @@ -2,7 +2,7 @@ def test_util_safe_path_list(): - res = dpath.util._split_path(["Ignore", "the/separator"], None) + res = dpath._split_path(["Ignore", "the/separator"], None) assert len(res) == 2 assert res[0] == "Ignore" From 9ef4bdef49c98053d11c25d663603410a96e71cd Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 14:50:54 +0200 Subject: [PATCH 26/37] Mock correct function --- tests/test_util_get_values.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_util_get_values.py b/tests/test_util_get_values.py index c72aa8f..a71b441 100644 --- a/tests/test_util_get_values.py +++ b/tests/test_util_get_values.py @@ -109,12 +109,12 @@ def test_values(): assert 2 in ret -@mock.patch('dpath.util.search') +@mock.patch('dpath.search') def test_values_passes_through(searchfunc): searchfunc.return_value = [] def y(): - pass + return False dpath.util.values({}, '/a/b', ':', y, False) searchfunc.assert_called_with({}, '/a/b', True, ':', y, False) From 9f06f0246696e055dc68ad4a619549ad5e233e46 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 15:10:34 +0200 Subject: [PATCH 27/37] PEP8 change --- dpath/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/types.py b/dpath/types.py index 2b44ce6..ed2c910 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -36,4 +36,4 @@ def creator( segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () - )""" \ No newline at end of file + )""" From b0681205b7a60018c468a88456347d91dde3a9cc Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 18:16:58 +0200 Subject: [PATCH 28/37] Enable DeprecationWarnings in tests --- tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e188ff3 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +import warnings + +warnings.simplefilter("always", DeprecationWarning) From eceb8cb3b2cb625da63deb71bbe6ecc3b2f762b6 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 18:37:12 +0200 Subject: [PATCH 29/37] Run flake8 separately from tests --- .github/workflows/tests.yml | 20 +++++++++++++++++++- flake8.ini | 5 +++++ tox.ini | 9 ++------- 3 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 flake8.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e7d924..3fa0f78 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,12 +38,30 @@ jobs: print(f'{hashseed=}') open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" + flake8: + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@main + + - name: Set up Python 3.10 + uses: actions/setup-python@main + + - name: Run Flake8 + uses: julianwachholz/flake8-action@v2 + with: + checkName: flake8 + config: flake8.ini + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Tests job tests: # The type of runner that the job will run on runs-on: ubuntu-latest - needs: generate-hashseed + needs: [generate-hashseed, flake8] strategy: matrix: diff --git a/flake8.ini b/flake8.ini new file mode 100644 index 0000000..92830d3 --- /dev/null +++ b/flake8.ini @@ -0,0 +1,5 @@ +[flake8] +filename= + setup.py, + dpath/, + tests/ diff --git a/tox.ini b/tox.ini index 94c441b..d613837 100644 --- a/tox.ini +++ b/tox.ini @@ -7,14 +7,14 @@ ignore = E501,E722 [tox] -envlist = pypy37, py38, py39, py310, flake8 +envlist = pypy37, py38, py39, py310 [gh-actions] python = pypy-3.7: pypy37 3.8: py38 3.9: py39 - 3.10: py310, flake8 + 3.10: py310 [testenv] deps = @@ -22,8 +22,3 @@ deps = mock nose2 commands = nose2 {posargs} - -[testenv:flake8] -deps = - flake8 -commands = flake8 setup.py dpath/ tests/ From ddf81a829041a019eeb912427d43091e15990b67 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 19:01:39 +0200 Subject: [PATCH 30/37] Working flake8 with annotations --- .github/workflows/tests.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3fa0f78..348a7fd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,14 +47,16 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@main - - - name: Run Flake8 - uses: julianwachholz/flake8-action@v2 with: - checkName: flake8 - config: flake8.ini - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + python-version: "3.10" + + - name: Setup flake8 annotations + uses: rbialon/flake8-annotations@v1.1 + + - name: Lint with flake8 + run: | + pip install flake8 + flake8 setup.py dpath/ tests/ # Tests job tests: From 29d344ebee3c17d6849522bea65eebd98e1dbfa2 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 19:10:15 +0200 Subject: [PATCH 31/37] Style fixes --- dpath/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dpath/types.py b/dpath/types.py index ed2c910..f56da31 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -7,11 +7,11 @@ class MergeType(IntFlag): """List objects are combined onto one long list (NOT a set). This is the default flag.""" REPLACE = auto() - """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination + """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination \ with the source.""" TYPESAFE = auto() - """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source + """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source \ replaces the destination in this situation.""" From b49e823dd07cf081f346fb47b0e1614e4019a825 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 19:14:51 +0200 Subject: [PATCH 32/37] Reorder tasks in workflow --- .github/workflows/tests.yml | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 348a7fd..ba8e2bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,23 +21,8 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # Generate a common hashseed for all tests - generate-hashseed: - runs-on: ubuntu-latest - - outputs: - hashseed: ${{ steps.generate.outputs.hashseed }} - - steps: - - name: Generate Hashseed - id: generate - run: | - python -c "import os - from random import randint - hashseed = randint(0, 4294967295) - print(f'{hashseed=}') - open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" + # Run flake8 linter flake8: runs-on: ubuntu-latest @@ -58,6 +43,23 @@ jobs: pip install flake8 flake8 setup.py dpath/ tests/ + # Generate a common hashseed for all tests + generate-hashseed: + runs-on: ubuntu-latest + + outputs: + hashseed: ${{ steps.generate.outputs.hashseed }} + + steps: + - name: Generate Hashseed + id: generate + run: | + python -c "import os + from random import randint + hashseed = randint(0, 4294967295) + print(f'{hashseed=}') + open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" + # Tests job tests: # The type of runner that the job will run on From 6460b645fcf30e8f8a3f88f73fe87cf2fd295ad8 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 19:15:02 +0200 Subject: [PATCH 33/37] Set minor version --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 4f7a825..127c148 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.8" +VERSION = "2.1.0" From 1c8fa849b1a0862643455f1751b699e23612198a Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 25 Nov 2022 15:22:01 +0200 Subject: [PATCH 34/37] Remove references to dpath.util in tests --- dpath/__init__.py | 2 +- dpath/util.py | 17 +++-- tests/test_broken_afilter.py | 18 +++--- tests/{test_util_delete.py => test_delete.py} | 10 +-- ..._util_get_values.py => test_get_values.py} | 64 +++++++++---------- tests/{test_util_merge.py => test_merge.py} | 34 +++++----- tests/{test_util_new.py => test_new.py} | 18 +++--- tests/{test_util_paths.py => test_paths.py} | 2 +- tests/{test_util_search.py => test_search.py} | 46 ++++++------- tests/{test_util_set.py => test_set.py} | 20 +++--- tests/test_types.py | 18 +++--- tests/test_unicode.py | 8 +-- 12 files changed, 128 insertions(+), 129 deletions(-) rename tests/{test_util_delete.py => test_delete.py} (79%) rename tests/{test_util_get_values.py => test_get_values.py} (62%) rename tests/{test_util_merge.py => test_merge.py} (78%) rename tests/{test_util_new.py => test_new.py} (80%) rename tests/{test_util_paths.py => test_paths.py} (91%) rename tests/{test_util_search.py => test_search.py} (72%) rename tests/{test_util_set.py => test_set.py} (74%) diff --git a/dpath/__init__.py b/dpath/__init__.py index a3180bd..79d33cf 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -186,7 +186,7 @@ def f(_, pair, results): raise KeyError(glob) elif len(results) > 1: - raise ValueError(f"dpath.util.get() globs must match only one leaf: {glob}") + raise ValueError(f"dpath.get() globs must match only one leaf: {glob}") return results[0] diff --git a/dpath/util.py b/dpath/util.py index 8d4a6b8..61e5580 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,9 +1,8 @@ import warnings -from typing import Any, Dict import dpath from dpath import _DEFAULT_SENTINEL -from dpath.types import Filter, Creator, MergeType +from dpath.types import MergeType def deprecated(func): @@ -18,35 +17,35 @@ def wrapper(*args, **kwargs): @deprecated -def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: +def new(obj, path, value, separator="/", creator=None): return dpath.new(obj, path, value, separator, creator) @deprecated -def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: +def delete(obj, glob, separator="/", afilter=None): return dpath.delete(obj, glob, separator, afilter) @deprecated -def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: +def set(obj, glob, value, separator="/", afilter=None): return dpath.set(obj, glob, value, separator, afilter) @deprecated -def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: +def get(obj, glob, separator="/", default=_DEFAULT_SENTINEL): return dpath.get(obj, glob, separator, default) @deprecated -def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): +def values(obj, glob, separator="/", afilter=None, dirs=True): return dpath.values(obj, glob, separator, afilter, dirs) @deprecated -def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): +def search(obj, glob, yielded=False, separator="/", afilter = None, dirs=True): return dpath.search(obj, glob, yielded, separator, afilter, dirs) @deprecated -def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): +def merge(dst, src, separator="/", afilter=None, flags=MergeType.ADDITIVE): return dpath.merge(dst, src, separator, afilter, flags), diff --git a/tests/test_broken_afilter.py b/tests/test_broken_afilter.py index 3e3a449..683c727 100644 --- a/tests/test_broken_afilter.py +++ b/tests/test_broken_afilter.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath import sys @@ -25,15 +25,15 @@ def afilter(x): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): + for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): assert path in paths - assert "view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a'] - assert "d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c'] + assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): + for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): assert path in paths - assert "view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a'] - assert "d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c'] + assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] def filter(x): sys.stderr.write(str(x)) @@ -52,7 +52,7 @@ def filter(x): ], } - results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', yielded=True)] - results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', afilter=filter, yielded=True)] + results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', yielded=True)] + results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', afilter=filter, yielded=True)] assert len(results) == 1 assert results[0][1]['type'] == 'correct' diff --git a/tests/test_util_delete.py b/tests/test_delete.py similarity index 79% rename from tests/test_util_delete.py rename to tests/test_delete.py index ecaa9b2..c7879b0 100644 --- a/tests/test_util_delete.py +++ b/tests/test_delete.py @@ -1,6 +1,6 @@ from nose2.tools.such import helper -import dpath.util +import dpath import dpath.exceptions @@ -11,7 +11,7 @@ def test_delete_separator(): }, } - dpath.util.delete(dict, ';a;b', separator=";") + dpath.delete(dict, ';a;b', separator=";") assert 'b' not in dict['a'] @@ -22,7 +22,7 @@ def test_delete_existing(): }, } - dpath.util.delete(dict, '/a/b') + dpath.delete(dict, '/a/b') assert 'b' not in dict['a'] @@ -33,7 +33,7 @@ def test_delete_missing(): } with helper.assertRaises(dpath.exceptions.PathNotFound): - dpath.util.delete(dict, '/a/b') + dpath.delete(dict, '/a/b') def test_delete_filter(): @@ -50,7 +50,7 @@ def afilter(x): }, } - dpath.util.delete(dict, '/a/*', afilter=afilter) + dpath.delete(dict, '/a/*', afilter=afilter) assert dict['a']['b'] == 0 assert dict['a']['c'] == 1 assert 'd' not in dict['a'] diff --git a/tests/test_util_get_values.py b/tests/test_get_values.py similarity index 62% rename from tests/test_util_get_values.py rename to tests/test_get_values.py index a71b441..9eeef82 100644 --- a/tests/test_util_get_values.py +++ b/tests/test_get_values.py @@ -5,16 +5,16 @@ import mock from nose2.tools.such import helper -import dpath.util +import dpath def test_util_get_root(): x = {'p': {'a': {'t': {'h': 'value'}}}} - ret = dpath.util.get(x, '/p/a/t/h') + ret = dpath.get(x, '/p/a/t/h') assert ret == 'value' - ret = dpath.util.get(x, '/') + ret = dpath.get(x, '/') assert ret == x @@ -31,11 +31,11 @@ def test_get_explicit_single(): }, } - assert dpath.util.get(ehash, '/a/b/c/f') == 2 - assert dpath.util.get(ehash, ['a', 'b', 'c', 'f']) == 2 - assert dpath.util.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2 - assert dpath.util.get(ehash, ['does', 'not', 'exist'], default=None) is None - assert dpath.util.get(ehash, ['doesnt', 'exist'], default=5) == 5 + assert dpath.get(ehash, '/a/b/c/f') == 2 + assert dpath.get(ehash, ['a', 'b', 'c', 'f']) == 2 + assert dpath.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2 + assert dpath.get(ehash, ['does', 'not', 'exist'], default=None) is None + assert dpath.get(ehash, ['doesnt', 'exist'], default=5) == 5 def test_get_glob_single(): @@ -51,10 +51,10 @@ def test_get_glob_single(): }, } - assert dpath.util.get(ehash, '/a/b/*/f') == 2 - assert dpath.util.get(ehash, ['a', 'b', '*', 'f']) == 2 - assert dpath.util.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2 - assert dpath.util.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6 + assert dpath.get(ehash, '/a/b/*/f') == 2 + assert dpath.get(ehash, ['a', 'b', '*', 'f']) == 2 + assert dpath.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2 + assert dpath.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6 def test_get_glob_multiple(): @@ -71,16 +71,16 @@ def test_get_glob_multiple(): }, } - helper.assertRaises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - helper.assertRaises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) - helper.assertRaises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd'], default=3) + helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd'], default=3) def test_get_absent(): ehash = {} - helper.assertRaises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - helper.assertRaises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) + helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) def test_values(): @@ -96,13 +96,13 @@ def test_values(): }, } - ret = dpath.util.values(ehash, '/a/b/c/*') + ret = dpath.values(ehash, '/a/b/c/*') assert isinstance(ret, list) assert 0 in ret assert 1 in ret assert 2 in ret - ret = dpath.util.values(ehash, ['a', 'b', 'c', '*']) + ret = dpath.values(ehash, ['a', 'b', 'c', '*']) assert isinstance(ret, list) assert 0 in ret assert 1 in ret @@ -116,17 +116,17 @@ def test_values_passes_through(searchfunc): def y(): return False - dpath.util.values({}, '/a/b', ':', y, False) + dpath.values({}, '/a/b', ':', y, False) searchfunc.assert_called_with({}, '/a/b', True, ':', y, False) - dpath.util.values({}, ['a', 'b'], ':', y, False) + dpath.values({}, ['a', 'b'], ':', y, False) searchfunc.assert_called_with({}, ['a', 'b'], True, ':', y, False) def test_none_values(): d = {'p': {'a': {'t': {'h': None}}}} - v = dpath.util.get(d, 'p/a/t/h') + v = dpath.get(d, 'p/a/t/h') assert v is None @@ -142,7 +142,7 @@ def test_values_list(): ], } - ret = dpath.util.values(a, 'actions/*') + ret = dpath.values(a, 'actions/*') assert isinstance(ret, list) assert len(ret) == 2 @@ -174,18 +174,18 @@ def func(x): } # It should be possible to get the callables: - assert dpath.util.get(testdict, 'a') == func - assert dpath.util.get(testdict, 'b')(42) == 42 + assert dpath.get(testdict, 'a') == func + assert dpath.get(testdict, 'b')(42) == 42 # It should be possible to get other values: - assert dpath.util.get(testdict, 'c/0') == testdict['c'][0] - assert dpath.util.get(testdict, 'd')[0] == testdict['d'][0] - assert dpath.util.get(testdict, 'd/0') == testdict['d'][0] - assert dpath.util.get(testdict, 'd/1') == testdict['d'][1] - assert dpath.util.get(testdict, 'e') == testdict['e'] + assert dpath.get(testdict, 'c/0') == testdict['c'][0] + assert dpath.get(testdict, 'd')[0] == testdict['d'][0] + assert dpath.get(testdict, 'd/0') == testdict['d'][0] + assert dpath.get(testdict, 'd/1') == testdict['d'][1] + assert dpath.get(testdict, 'e') == testdict['e'] # Values should also still work: - assert dpath.util.values(testdict, 'f/config') == ['something'] + assert dpath.values(testdict, 'f/config') == ['something'] # Data classes should also be retrievable: try: @@ -207,4 +207,4 @@ class Connection: ), } - assert dpath.util.search(testdict, 'g/my*')['g']['my-key'] == testdict['g']['my-key'] + assert dpath.search(testdict, 'g/my*')['g']['my-key'] == testdict['g']['my-key'] diff --git a/tests/test_util_merge.py b/tests/test_merge.py similarity index 78% rename from tests/test_util_merge.py rename to tests/test_merge.py index 695e58f..a8b638c 100644 --- a/tests/test_util_merge.py +++ b/tests/test_merge.py @@ -3,7 +3,7 @@ from nose2.tools.such import helper -import dpath.util +import dpath from dpath import MergeType @@ -20,7 +20,7 @@ def test_merge_typesafe_and_separator(): } try: - dpath.util.merge(dst, src, flags=(dpath.util.MergeType.ADDITIVE | dpath.util.MergeType.TYPESAFE), separator=";") + dpath.merge(dst, src, flags=(dpath.MergeType.ADDITIVE | dpath.MergeType.TYPESAFE), separator=";") except TypeError as e: assert str(e).endswith("dict;integer") @@ -36,7 +36,7 @@ def test_merge_simple_int(): "integer": 3, } - dpath.util.merge(dst, src) + dpath.merge(dst, src) assert dst["integer"] == src["integer"], "%r != %r" % (dst["integer"], src["integer"]) @@ -48,7 +48,7 @@ def test_merge_simple_string(): "string": "lol I am a string", } - dpath.util.merge(dst, src) + dpath.merge(dst, src) assert dst["string"] == src["string"], "%r != %r" % (dst["string"], src["string"]) @@ -60,7 +60,7 @@ def test_merge_simple_list_additive(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=MergeType.ADDITIVE) + dpath.merge(dst, src, flags=MergeType.ADDITIVE) assert dst["list"] == [0, 1, 2, 3, 7, 8, 9, 10], "%r != %r" % (dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) @@ -72,7 +72,7 @@ def test_merge_simple_list_replace(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MergeType.REPLACE) + dpath.merge(dst, src, flags=dpath.MergeType.REPLACE) assert dst["list"] == [7, 8, 9, 10], "%r != %r" % (dst["list"], [7, 8, 9, 10]) @@ -88,7 +88,7 @@ def test_merge_simple_dict(): }, } - dpath.util.merge(dst, src) + dpath.merge(dst, src) assert dst["dict"]["key"] == src["dict"]["key"], "%r != %r" % (dst["dict"]["key"], src["dict"]["key"]) @@ -107,7 +107,7 @@ def afilter(x): } dst = {} - dpath.util.merge(dst, src, afilter=afilter) + dpath.merge(dst, src, afilter=afilter) assert "key2" in dst assert "key" not in dst assert "otherdict" not in dst @@ -123,7 +123,7 @@ def test_merge_typesafe(): ], } - helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MergeType.TYPESAFE) + helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) def test_merge_mutables(): @@ -149,26 +149,26 @@ class tcis(list): "ms": tcis(['a', 'b', 'c']), } - dpath.util.merge(dst, src) + dpath.merge(dst, src) print(dst) assert dst["mm"]["a"] == src["mm"]["a"] assert dst['ms'][2] == 'c' assert "casserole" in dst["mm"] - helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MergeType.TYPESAFE) + helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) def test_merge_replace_1(): dct_a = {"a": {"b": [1, 2, 3]}} dct_b = {"a": {"b": [1]}} - dpath.util.merge(dct_a, dct_b, flags=dpath.util.MergeType.REPLACE) + dpath.merge(dct_a, dct_b, flags=dpath.MergeType.REPLACE) assert len(dct_a['a']['b']) == 1 def test_merge_replace_2(): d1 = {'a': [0, 1, 2]} d2 = {'a': ['a']} - dpath.util.merge(d1, d2, flags=dpath.util.MergeType.REPLACE) + dpath.merge(d1, d2, flags=dpath.MergeType.REPLACE) assert len(d1['a']) == 1 assert d1['a'][0] == 'a' @@ -180,18 +180,18 @@ def test_merge_list(): dst1 = {} for d in [copy.deepcopy(src), copy.deepcopy(p1)]: - dpath.util.merge(dst1, d) + dpath.merge(dst1, d) dst2 = {} for d in [copy.deepcopy(src), copy.deepcopy(p2)]: - dpath.util.merge(dst2, d) + dpath.merge(dst2, d) assert dst1["l"] == [1, 2] assert dst2["l"] == [1] dst1 = {} for d in [src, p1]: - dpath.util.merge(dst1, d) + dpath.merge(dst1, d) dst2 = {} for d in [src, p2]: - dpath.util.merge(dst2, d) + dpath.merge(dst2, d) assert dst1["l"] == [1, 2] assert dst2["l"] == [1, 2] diff --git a/tests/test_util_new.py b/tests/test_new.py similarity index 80% rename from tests/test_util_new.py rename to tests/test_new.py index 3c2a3c4..6da31e7 100644 --- a/tests/test_util_new.py +++ b/tests/test_new.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath def test_set_new_separator(): @@ -7,10 +7,10 @@ def test_set_new_separator(): }, } - dpath.util.new(dict, ';a;b', 1, separator=";") + dpath.new(dict, ';a;b', 1, separator=";") assert dict['a']['b'] == 1 - dpath.util.new(dict, ['a', 'b'], 1, separator=";") + dpath.new(dict, ['a', 'b'], 1, separator=";") assert dict['a']['b'] == 1 @@ -20,10 +20,10 @@ def test_set_new_dict(): }, } - dpath.util.new(dict, '/a/b', 1) + dpath.new(dict, '/a/b', 1) assert dict['a']['b'] == 1 - dpath.util.new(dict, ['a', 'b'], 1) + dpath.new(dict, ['a', 'b'], 1) assert dict['a']['b'] == 1 @@ -33,11 +33,11 @@ def test_set_new_list(): ], } - dpath.util.new(dict, '/a/1', 1) + dpath.new(dict, '/a/1', 1) assert dict['a'][1] == 1 assert dict['a'][0] is None - dpath.util.new(dict, ['a', 1], 1) + dpath.new(dict, ['a', 1], 1) assert dict['a'][1] == 1 assert dict['a'][0] is None @@ -49,7 +49,7 @@ def test_set_new_list_path_with_separator(): }, } - dpath.util.new(dict, ['a', 'b/c/d', 0], 1) + dpath.new(dict, ['a', 'b/c/d', 0], 1) assert len(dict['a']) == 1 assert len(dict['a']['b/c/d']) == 1 assert dict['a']['b/c/d'][0] == 1 @@ -76,7 +76,7 @@ def mycreator(obj, pathcomp, nextpathcomp, hints): obj[target] = {} print(obj) - dpath.util.new(d, '/a/2', 3, creator=mycreator) + dpath.new(d, '/a/2', 3, creator=mycreator) print(d) assert isinstance(d['a'], list) assert len(d['a']) == 3 diff --git a/tests/test_util_paths.py b/tests/test_paths.py similarity index 91% rename from tests/test_util_paths.py rename to tests/test_paths.py index 49bec1b..c63d728 100644 --- a/tests/test_util_paths.py +++ b/tests/test_paths.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath def test_util_safe_path_list(): diff --git a/tests/test_util_search.py b/tests/test_search.py similarity index 72% rename from tests/test_util_search.py rename to tests/test_search.py index a974963..5830088 100644 --- a/tests/test_util_search.py +++ b/tests/test_search.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath def test_search_paths_with_separator(): @@ -22,10 +22,10 @@ def test_search_paths_with_separator(): 'a;b;c;f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True, separator=";"): + for (path, value) in dpath.search(dict, '/**', yielded=True, separator=";"): assert path in paths - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, separator=";"): + for (path, value) in dpath.search(dict, ['**'], yielded=True, separator=";"): assert path in paths @@ -50,10 +50,10 @@ def test_search_paths(): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True): + for (path, value) in dpath.search(dict, '/**', yielded=True): assert path in paths - for (path, value) in dpath.util.search(dict, ['**'], yielded=True): + for (path, value) in dpath.search(dict, ['**'], yielded=True): assert path in paths @@ -80,15 +80,15 @@ def afilter(x): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): + for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): assert path in paths - assert "view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a'] - assert "d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c'] + assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): + for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): assert path in paths - assert "view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a'] - assert "d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c'] + assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] def test_search_globbing(): @@ -108,10 +108,10 @@ def test_search_globbing(): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/a/**/[df]', yielded=True): + for (path, value) in dpath.search(dict, '/a/**/[df]', yielded=True): assert path in paths - for (path, value) in dpath.util.search(dict, ['a', '**', '[df]'], yielded=True): + for (path, value) in dpath.search(dict, ['a', '**', '[df]'], yielded=True): assert path in paths @@ -125,12 +125,12 @@ def test_search_return_dict_head(): }, }, } - res = dpath.util.search(tdict, '/a/b') + res = dpath.search(tdict, '/a/b') assert isinstance(res['a']['b'], dict) assert len(res['a']['b']) == 3 assert res['a']['b'] == {0: 0, 1: 1, 2: 2} - res = dpath.util.search(tdict, ['a', 'b']) + res = dpath.search(tdict, ['a', 'b']) assert isinstance(res['a']['b'], dict) assert len(res['a']['b']) == 3 assert res['a']['b'] == {0: 0, 1: 1, 2: 2} @@ -147,12 +147,12 @@ def test_search_return_dict_globbed(): }, } - res = dpath.util.search(tdict, '/a/b/[02]') + res = dpath.search(tdict, '/a/b/[02]') assert isinstance(res['a']['b'], dict) assert len(res['a']['b']) == 2 assert res['a']['b'] == {0: 0, 2: 2} - res = dpath.util.search(tdict, ['a', 'b', '[02]']) + res = dpath.search(tdict, ['a', 'b', '[02]']) assert isinstance(res['a']['b'], dict) assert len(res['a']['b']) == 2 assert res['a']['b'] == {0: 0, 2: 2} @@ -169,12 +169,12 @@ def test_search_return_list_head(): }, } - res = dpath.util.search(tdict, '/a/b') + res = dpath.search(tdict, '/a/b') assert isinstance(res['a']['b'], list) assert len(res['a']['b']) == 3 assert res['a']['b'] == [0, 1, 2] - res = dpath.util.search(tdict, ['a', 'b']) + res = dpath.search(tdict, ['a', 'b']) assert isinstance(res['a']['b'], list) assert len(res['a']['b']) == 3 assert res['a']['b'] == [0, 1, 2] @@ -191,12 +191,12 @@ def test_search_return_list_globbed(): } } - res = dpath.util.search(tdict, '/a/b/[02]') + res = dpath.search(tdict, '/a/b/[02]') assert isinstance(res['a']['b'], list) assert len(res['a']['b']) == 3 assert res['a']['b'] == [0, None, 2] - res = dpath.util.search(tdict, ['a', 'b', '[02]']) + res = dpath.search(tdict, ['a', 'b', '[02]']) assert isinstance(res['a']['b'], list) assert len(res['a']['b']) == 3 assert res['a']['b'] == [0, None, 2] @@ -212,7 +212,7 @@ def test_search_list_key_with_separator(): }, } - res = dpath.util.search(tdict, ['a', '/b/d']) + res = dpath.search(tdict, ['a', '/b/d']) assert 'b' not in res['a'] assert res['a']['/b/d'] == 'success' @@ -231,7 +231,7 @@ def test_search_multiple_stars(): } testpath = 'a/*/b/*/c' - res = dpath.util.search(testdata, testpath) + res = dpath.search(testdata, testpath) assert len(res['a'][0]['b']) == 3 assert res['a'][0]['b'][0]['c'] == 1 assert res['a'][0]['b'][1]['c'] == 2 diff --git a/tests/test_util_set.py b/tests/test_set.py similarity index 74% rename from tests/test_util_set.py rename to tests/test_set.py index 1592590..ef2dd96 100644 --- a/tests/test_util_set.py +++ b/tests/test_set.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath def test_set_existing_separator(): @@ -8,11 +8,11 @@ def test_set_existing_separator(): }, } - dpath.util.set(dict, ';a;b', 1, separator=";") + dpath.set(dict, ';a;b', 1, separator=";") assert dict['a']['b'] == 1 dict['a']['b'] = 0 - dpath.util.set(dict, ['a', 'b'], 1, separator=";") + dpath.set(dict, ['a', 'b'], 1, separator=";") assert dict['a']['b'] == 1 @@ -23,11 +23,11 @@ def test_set_existing_dict(): }, } - dpath.util.set(dict, '/a/b', 1) + dpath.set(dict, '/a/b', 1) assert dict['a']['b'] == 1 dict['a']['b'] = 0 - dpath.util.set(dict, ['a', 'b'], 1) + dpath.set(dict, ['a', 'b'], 1) assert dict['a']['b'] == 1 @@ -38,11 +38,11 @@ def test_set_existing_list(): ], } - dpath.util.set(dict, '/a/0', 1) + dpath.set(dict, '/a/0', 1) assert dict['a'][0] == 1 dict['a'][0] = 0 - dpath.util.set(dict, ['a', '0'], 1) + dpath.set(dict, ['a', '0'], 1) assert dict['a'][0] == 1 @@ -60,7 +60,7 @@ def afilter(x): } } - dpath.util.set(dict, '/a/*', 31337, afilter=afilter) + dpath.set(dict, '/a/*', 31337, afilter=afilter) assert dict['a']['b'] == 0 assert dict['a']['c'] == 1 assert dict['a']['d'] == 31337 @@ -73,7 +73,7 @@ def afilter(x): } } - dpath.util.set(dict, ['a', '*'], 31337, afilter=afilter) + dpath.set(dict, ['a', '*'], 31337, afilter=afilter) assert dict['a']['b'] == 0 assert dict['a']['c'] == 1 assert dict['a']['d'] == 31337 @@ -86,6 +86,6 @@ def test_set_existing_path_with_separator(): }, } - dpath.util.set(dict, ['a', 'b/c/d'], 1) + dpath.set(dict, ['a', 'b/c/d'], 1) assert len(dict['a']) == 1 assert dict['a']['b/c/d'] == 1 diff --git a/tests/test_types.py b/tests/test_types.py index 56eb98f..aa613ed 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -2,7 +2,7 @@ from nose2.tools.such import helper -import dpath.util +import dpath from dpath import MergeType @@ -65,12 +65,12 @@ def append(self, value): def test_types_set(): data = TestMapping({"a": TestSequence([0])}) - dpath.util.set(data, '/a/0', 1) + dpath.set(data, '/a/0', 1) assert data['a'][0] == 1 data['a'][0] = 0 - dpath.util.set(data, ['a', '0'], 1) + dpath.set(data, ['a', '0'], 1) assert data['a'][0] == 1 @@ -100,14 +100,14 @@ def test_types_merge_simple_list_replace(): "list": TestSequence([0, 1, 2, 3]) }) - dpath.util.merge(dst, src, flags=MergeType.REPLACE) + dpath.merge(dst, src, flags=MergeType.REPLACE) assert dst["list"] == TestSequence([7, 8, 9, 10]), "%r != %r" % (dst["list"], TestSequence([7, 8, 9, 10])) def test_types_get_absent(): ehash = TestMapping() - helper.assertRaises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - helper.assertRaises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) + helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) def test_types_get_glob_multiple(): @@ -124,8 +124,8 @@ def test_types_get_glob_multiple(): }), }) - helper.assertRaises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - helper.assertRaises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) def test_delete_filter(): @@ -142,7 +142,7 @@ def afilter(x): }), }) - dpath.util.delete(data, '/a/*', afilter=afilter) + dpath.delete(data, '/a/*', afilter=afilter) assert data['a']['b'] == 0 assert data['a']['c'] == 1 assert 'd' not in data['a'] diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 09bc4db..d4e8033 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,11 +1,11 @@ -import dpath.util +import dpath def test_unicode_merge(): a = {'中': 'zhong'} b = {'文': 'wen'} - dpath.util.merge(a, b) + dpath.merge(a, b) assert len(a.keys()) == 2 assert a['中'] == 'zhong' assert a['文'] == 'wen' @@ -14,7 +14,7 @@ def test_unicode_merge(): def test_unicode_search(): a = {'中': 'zhong'} - results = [[x[0], x[1]] for x in dpath.util.search(a, '*', yielded=True)] + results = [[x[0], x[1]] for x in dpath.search(a, '*', yielded=True)] assert len(results) == 1 assert results[0][0] == '中' assert results[0][1] == 'zhong' @@ -24,7 +24,7 @@ def test_unicode_str_hybrid(): a = {'first': u'1'} b = {u'second': '2'} - dpath.util.merge(a, b) + dpath.merge(a, b) assert len(a.keys()) == 2 assert a[u'second'] == '2' assert a['second'] == u'2' From 17eab50391d6882eb1692ea30729facd55d11ae7 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 25 Nov 2022 15:25:21 +0200 Subject: [PATCH 35/37] PEP8 fix --- dpath/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/util.py b/dpath/util.py index 61e5580..60d0319 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -42,7 +42,7 @@ def values(obj, glob, separator="/", afilter=None, dirs=True): @deprecated -def search(obj, glob, yielded=False, separator="/", afilter = None, dirs=True): +def search(obj, glob, yielded=False, separator="/", afilter=None, dirs=True): return dpath.search(obj, glob, yielded, separator, afilter, dirs) From 36a6f5f4e802282e6a0f30a8c99525bf08ccc654 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 25 Nov 2022 15:37:11 +0200 Subject: [PATCH 36/37] Add note regarding 3.6 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 4b0f386..a6891d6 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ # (Source: https://github.com/hugovk/pypi-tools/blob/master/README.md) # # Conclusion: In order to accommodate type hinting support must be limited to Python versions >=3.6. + # 3.6 was dropped because of EOL and this issue: https://github.com/actions/setup-python/issues/544 python_requires=">=3.7", classifiers=[ 'Development Status :: 5 - Production/Stable', From 7a565ee55ae02a476e8df0ce1fc69d50ac27297e Mon Sep 17 00:00:00 2001 From: moomoohk Date: Mon, 28 Nov 2022 01:11:32 +0200 Subject: [PATCH 37/37] Type hint improvements and exports --- dpath/__init__.py | 32 ++++++++++++++++++++++---------- dpath/segments.py | 8 ++++---- dpath/types.py | 12 +++++++++--- tests/test_types.py | 10 ++++++++-- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index 79d33cf..c717314 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -11,19 +11,26 @@ "segments", "types", "version", + "MergeType", + "PathSegment", + "Filter", + "Glob", + "Path", + "Hints", + "Creator", ] from collections.abc import MutableMapping, MutableSequence -from typing import Union, List, Dict, Any +from typing import Union, List, Any, Callable, Optional from dpath import segments, options from dpath.exceptions import InvalidKeyName, PathNotFound -from dpath.types import MergeType, PathSegment, Creator, Filter +from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, Path, Hints _DEFAULT_SENTINEL = object() -def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: +def _split_path(path: Path, separator: Optional[str]) -> Union[List[PathSegment], PathSegment]: """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -51,7 +58,7 @@ def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegme return split_segments -def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: +def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator = None) -> MutableMapping: """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only @@ -71,7 +78,7 @@ def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> return segments.set(obj, split_segments, value) -def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: +def delete(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None) -> int: """ Given a obj, delete all elements that match the glob. @@ -130,7 +137,7 @@ def f(obj, pair, counter): return deleted -def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: +def set(obj: MutableMapping, glob: Glob, value, separator="/", afilter: Filter = None) -> int: """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. @@ -155,7 +162,12 @@ def f(obj, pair, counter): return changed -def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: +def get( + obj: MutableMapping, + glob: Glob, + separator="/", + default: Any = _DEFAULT_SENTINEL +) -> Union[MutableMapping, object, Callable]: """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. @@ -191,7 +203,7 @@ def f(_, pair, results): return results[0] -def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): +def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None, dirs=True): """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). @@ -201,7 +213,7 @@ def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=Tru return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] -def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): +def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilter: Filter = None, dirs=True): """ Given a path glob, return a dictionary containing all keys that matched the given glob. @@ -243,7 +255,7 @@ def f(obj, pair, result): return segments.fold(obj, f, {}) -def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): +def merge(dst: MutableMapping, src: MutableMapping, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): """ Merge source into destination. Like dict.update() but performs deep merging. diff --git a/dpath/segments.py b/dpath/segments.py index f452d12..d87a7b2 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,6 +1,6 @@ from copy import deepcopy from fnmatch import fnmatchcase -from typing import List, Sequence, Tuple, Iterator, Any, Dict, Union, Optional +from typing import List, Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound @@ -266,7 +266,7 @@ def extend(thing: List, index: int, value=None): def _default_creator( - current: Union[Dict, List], + current: Union[MutableMapping, List], segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () @@ -301,12 +301,12 @@ def _default_creator( def set( - obj, + obj: MutableMapping, segments: Sequence[PathSegment], value, creator: Optional[Creator] = _default_creator, hints: Hints = () -): +) -> MutableMapping: """ Set the value in obj at the place indicated by segments. If creator is not None (default __default_creator__), then call the creator function to diff --git a/dpath/types.py b/dpath/types.py index f56da31..b876e6a 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -1,5 +1,5 @@ from enum import IntFlag, auto -from typing import Union, Any, Callable, Sequence, Tuple, Dict, List, Optional +from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping class MergeType(IntFlag): @@ -23,16 +23,22 @@ class MergeType(IntFlag): (Any) -> bool""" +Glob = Union[str, Sequence[str]] +"""Type alias for glob parameters.""" + +Path = Union[str, Sequence[PathSegment]] +"""Type alias for path parameters.""" + Hints = Sequence[Tuple[PathSegment, type]] """Type alias for creator function hint sequences.""" -Creator = Callable[[Union[Dict, List], Sequence[PathSegment], int, Optional[Hints]], None] +Creator = Callable[[Union[MutableMapping, List], Path, int, Optional[Hints]], None] """Type alias for creator functions. Example creator function signature: def creator( - current: Union[Dict, List], + current: Union[MutableMapping, List], segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () diff --git a/tests/test_types.py b/tests/test_types.py index aa613ed..39993f3 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -7,7 +7,10 @@ class TestMapping(MutableMapping): - def __init__(self, data={}): + def __init__(self, data=None): + if data is None: + data = {} + self._mapping = {} self._mapping.update(data) @@ -31,7 +34,10 @@ def __delitem__(self, key): class TestSequence(MutableSequence): - def __init__(self, data=list()): + def __init__(self, data=None): + if data is None: + data = list() + self._list = [] + data def __len__(self):