From 4a28d52e140a1a33bf322f543ad0dfba9acb1584 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 17:18:32 +0200 Subject: [PATCH 01/14] Support negative indexes --- dpath/segments.py | 24 +++++++++++++++++------- dpath/types.py | 22 ++++++++++++++++++++++ tests/test_search.py | 17 +++++++++++++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index faa763f..baa07c8 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -4,7 +4,7 @@ from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath.types import PathSegment, Creator, Hints +from dpath.types import PathSegment, Creator, Hints, Glob, Path, CyclicInt def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: @@ -21,7 +21,10 @@ def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: return iter(node.items()) except AttributeError: try: - return zip(range(len(node)), node) + indices = range(len(node)) + # Make all list indices cyclic so negative (wraparound) indexes are supported + indices = map(lambda i: CyclicInt(i, len(node)), indices) + return zip(indices, node) except TypeError: # This can happen in cases where the node isn't leaf(node) == True, # but also isn't actually iterable. Instead of this being an error @@ -163,7 +166,7 @@ class Star(object): STAR = Star() -def match(segments: Sequence[PathSegment], glob: Sequence[str]): +def match(segments: Path, glob: Glob): """ Return True if the segments match the given glob, otherwise False. @@ -214,7 +217,9 @@ def match(segments: Sequence[PathSegment], glob: Sequence[str]): # If we were successful in matching up the lengths, then we can # compare them using fnmatch. if path_len == len(ss_glob): - for s, g in zip(map(int_str, segments), map(int_str, ss_glob)): + # TODO: Delete if not needed (previous code) - i = zip(map(int_str, segments), map(int_str, ss_glob)) + i = zip(segments, ss_glob) + for s, g in i: # Match the stars we added to the glob to the type of the # segment itself. if g is STAR: @@ -223,10 +228,15 @@ def match(segments: Sequence[PathSegment], glob: Sequence[str]): else: g = '*' - # Let's see if the glob matches. We will turn any kind of - # exception while attempting to match into a False for the - # match. try: + # If search path segment (s) is an int and the current evaluated index (g) is int-like, + # then g is surely a sequence index. Convert it to int and compare. + if isinstance(s, int) and isinstance(g, str) and (g.count("-") == 0 or g.lstrip("-").isdigit()): + return s == int(g) + + # Let's see if the glob matches. We will turn any kind of + # exception while attempting to match into a False for the + # match. if not fnmatchcase(s, g): return False except: diff --git a/dpath/types.py b/dpath/types.py index b876e6a..1890a03 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -2,6 +2,28 @@ from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping +class CyclicInt(int): + """Same as a normal int but mimicks the behavior of list indexes (can be compared to a negative number)""" + + def __new__(cls, value, max_value, *args, **kwargs): + if value >= max_value: + raise TypeError( + f"Tried to initiate a CyclicInt with a value ({value}) " + f"greater than the provided max value ({max_value})" + ) + + obj = super().__new__(cls, value) + obj.max_value = max_value + + return obj + + def __eq__(self, other): + return int(self) == (self.max_value + other) % self.max_value + + def __repr__(self): + return f"" + + class MergeType(IntFlag): ADDITIVE = auto() """List objects are combined onto one long list (NOT a set). This is the default flag.""" diff --git a/tests/test_search.py b/tests/test_search.py index 5830088..d274d3d 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -236,3 +236,20 @@ def test_search_multiple_stars(): assert res['a'][0]['b'][0]['c'] == 1 assert res['a'][0]['b'][1]['c'] == 2 assert res['a'][0]['b'][2]['c'] == 3 + + +def test_search_glob_list(): + d = {'a': {'b': []}} + res = dpath.search(d, 'a/b/*') + assert res == {'a': {'b': []}} + + d = {'a': {'b': [1, 2, 3]}} + dpath.search(d, 'a/b/*', afilter=lambda x: x > 3 if isinstance(x, int) else True) + assert res == {'a': {'b': []}} + + +def test_search_negative_index(): + d = {'a': {'b': [1, 2, 3]}} + res = dpath.search(d, 'a/b/-1') + + assert res == dpath.search(d, "a/b/2") From 308b9779e668df31d80a3060d451b90d7e022551 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 17:43:19 +0200 Subject: [PATCH 02/14] Minor improvements --- dpath/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dpath/types.py b/dpath/types.py index 1890a03..1cd7242 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -3,9 +3,9 @@ class CyclicInt(int): - """Same as a normal int but mimicks the behavior of list indexes (can be compared to a negative number)""" + """Same as a normal int but mimicks the behavior of list indexes (can be compared to a negative number).""" - def __new__(cls, value, max_value, *args, **kwargs): + def __new__(cls, value: int, max_value: int, *args, **kwargs): if value >= max_value: raise TypeError( f"Tried to initiate a CyclicInt with a value ({value}) " From cb6b94b87a5e7cdc733e1882825aa5a3bced3af6 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 17:48:27 +0200 Subject: [PATCH 03/14] Improve negative number check --- dpath/segments.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index baa07c8..ece8230 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -230,9 +230,12 @@ def match(segments: Path, glob: Glob): try: # If search path segment (s) is an int and the current evaluated index (g) is int-like, - # then g is surely a sequence index. Convert it to int and compare. - if isinstance(s, int) and isinstance(g, str) and (g.count("-") == 0 or g.lstrip("-").isdigit()): - return s == int(g) + # then g is surely a sequence index as well. Convert it to int and compare. + if isinstance(s, int) and isinstance(g, str): + neg_c = g.count("-") + + if (neg_c == 0 and g.isdigit()) or (neg_c == 1 and g.lstrip("-").isdigit()): + return s == int(g) # Let's see if the glob matches. We will turn any kind of # exception while attempting to match into a False for the From 42ad0a1a0e75987d6fbdb50671ddd51140d712bc Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 17:58:49 +0200 Subject: [PATCH 04/14] Remove unnecessary negative number check --- dpath/segments.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index ece8230..6907a85 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -232,11 +232,12 @@ def match(segments: Path, glob: Glob): # If search path segment (s) is an int and the current evaluated index (g) is int-like, # then g is surely a sequence index as well. Convert it to int and compare. if isinstance(s, int) and isinstance(g, str): - neg_c = g.count("-") - - if (neg_c == 0 and g.isdigit()) or (neg_c == 1 and g.lstrip("-").isdigit()): - return s == int(g) + return s == int(g) + except: + # Will reach this point if g can't be converted to an int... + pass + try: # Let's see if the glob matches. We will turn any kind of # exception while attempting to match into a False for the # match. From 869fe2d6c5fc10397f054796438251bbf0ee130f Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 18:05:06 +0200 Subject: [PATCH 05/14] Fix values to work with fnmatchcase --- dpath/segments.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 6907a85..1e5dc04 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -230,12 +230,13 @@ def match(segments: Path, glob: Glob): try: # If search path segment (s) is an int and the current evaluated index (g) is int-like, - # then g is surely a sequence index as well. Convert it to int and compare. + # then g might be a sequence index as well. Try converting it to an int. if isinstance(s, int) and isinstance(g, str): return s == int(g) except: - # Will reach this point if g can't be converted to an int... - pass + # Will reach this point if g can't be converted to an int (e.g. when g is a RegEx pattern). + # In this case convert s to a str so fnmatch can work on it. + s = str(s) try: # Let's see if the glob matches. We will turn any kind of From f15da4faa1c6618bf259b58d75dade0a3306efcc Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 11:52:23 +0200 Subject: [PATCH 06/14] Add str overload to CyclicInt --- dpath/types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dpath/types.py b/dpath/types.py index 1cd7242..1975dc1 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -23,6 +23,9 @@ def __eq__(self, other): def __repr__(self): return f"" + def __str__(self): + return str(int(self)) + class MergeType(IntFlag): ADDITIVE = auto() From 49e06d92718bba5f6f5f9136edace7051ec714f8 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:13:25 +0200 Subject: [PATCH 07/14] Simplify int handling in matching code --- dpath/segments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 1e5dc04..55914c5 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -229,9 +229,9 @@ def match(segments: Path, glob: Glob): g = '*' try: - # If search path segment (s) is an int and the current evaluated index (g) is int-like, - # then g might be a sequence index as well. Try converting it to an int. - if isinstance(s, int) and isinstance(g, str): + # If search path segment (s) is an int then assume currently evaluated index (g) might be a sequenc + # index as well. Try converting it to an int. + if isinstance(s, int): return s == int(g) except: # Will reach this point if g can't be converted to an int (e.g. when g is a RegEx pattern). From 6572e591b14dd0831e45dfeb8c24c186d96727de Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:13:30 +0200 Subject: [PATCH 08/14] Remove test case --- tests/test_search.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index d274d3d..7219d2b 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -238,16 +238,6 @@ def test_search_multiple_stars(): assert res['a'][0]['b'][2]['c'] == 3 -def test_search_glob_list(): - d = {'a': {'b': []}} - res = dpath.search(d, 'a/b/*') - assert res == {'a': {'b': []}} - - d = {'a': {'b': [1, 2, 3]}} - dpath.search(d, 'a/b/*', afilter=lambda x: x > 3 if isinstance(x, int) else True) - assert res == {'a': {'b': []}} - - def test_search_negative_index(): d = {'a': {'b': [1, 2, 3]}} res = dpath.search(d, 'a/b/-1') From 6c513fa673883630da9591a5a61786903f86f993 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:15:36 +0200 Subject: [PATCH 09/14] Bump version --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 5b0431e..b777579 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.1" +VERSION = "2.1.2" From e37fdad711d1c3b720a331b1279171a37f612977 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:28:20 +0200 Subject: [PATCH 10/14] Continue evaluating entire path when handling int --- dpath/segments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 55914c5..ac7d390 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -229,10 +229,10 @@ def match(segments: Path, glob: Glob): g = '*' try: - # If search path segment (s) is an int then assume currently evaluated index (g) might be a sequenc + # If search path segment (s) is an int then assume currently evaluated index (g) might be a sequence # index as well. Try converting it to an int. - if isinstance(s, int): - return s == int(g) + if isinstance(s, int) and s == int(g): + continue except: # Will reach this point if g can't be converted to an int (e.g. when g is a RegEx pattern). # In this case convert s to a str so fnmatch can work on it. From 58407dbda7fcd915df80dc206f48432ac2503846 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 14:34:51 +0200 Subject: [PATCH 11/14] Add type hints --- dpath/segments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/segments.py b/dpath/segments.py index ac7d390..b2f55fd 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -406,7 +406,7 @@ def foldm(obj, f, acc): return acc -def view(obj, glob): +def view(obj: MutableMapping, glob: Glob): """ Return a view of the object where the glob matches. A view retains the same form as the obj, but is limited to only the paths that From 317e3cd5f680b1543d8b511fad1a40db922caf4c Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 14:35:01 +0200 Subject: [PATCH 12/14] Improve CyclicInt type --- dpath/types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dpath/types.py b/dpath/types.py index 1975dc1..6906224 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -18,10 +18,13 @@ def __new__(cls, value: int, max_value: int, *args, **kwargs): return obj def __eq__(self, other): + if not isinstance(other, int): + return False + return int(self) == (self.max_value + other) % self.max_value def __repr__(self): - return f"" + return f"" def __str__(self): return str(int(self)) From 66a64a390544ae9014884524941844410970ecbe Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sat, 3 Dec 2022 22:30:15 +0200 Subject: [PATCH 13/14] Rename CyclicInt to SymmetricInt --- dpath/segments.py | 7 +++---- dpath/types.py | 9 ++++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index b2f55fd..7a48817 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -4,7 +4,7 @@ from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath.types import PathSegment, Creator, Hints, Glob, Path, CyclicInt +from dpath.types import PathSegment, Creator, Hints, Glob, Path, SymmetricInt def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: @@ -22,8 +22,8 @@ def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: except AttributeError: try: indices = range(len(node)) - # Make all list indices cyclic so negative (wraparound) indexes are supported - indices = map(lambda i: CyclicInt(i, len(node)), indices) + # Convert all list indices to object so negative indexes are supported. + indices = map(lambda i: SymmetricInt(i, len(node)), indices) return zip(indices, node) except TypeError: # This can happen in cases where the node isn't leaf(node) == True, @@ -217,7 +217,6 @@ def match(segments: Path, glob: Glob): # If we were successful in matching up the lengths, then we can # compare them using fnmatch. if path_len == len(ss_glob): - # TODO: Delete if not needed (previous code) - i = zip(map(int_str, segments), map(int_str, ss_glob)) i = zip(segments, ss_glob) for s, g in i: # Match the stars we added to the glob to the type of the diff --git a/dpath/types.py b/dpath/types.py index 6906224..210b24a 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -2,13 +2,13 @@ from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping -class CyclicInt(int): +class SymmetricInt(int): """Same as a normal int but mimicks the behavior of list indexes (can be compared to a negative number).""" def __new__(cls, value: int, max_value: int, *args, **kwargs): if value >= max_value: raise TypeError( - f"Tried to initiate a CyclicInt with a value ({value}) " + f"Tried to initiate a {cls.__name__} with a value ({value}) " f"greater than the provided max value ({max_value})" ) @@ -21,10 +21,13 @@ def __eq__(self, other): if not isinstance(other, int): return False + if other >= self.max_value or other >= -self.max_value: + return False + return int(self) == (self.max_value + other) % self.max_value def __repr__(self): - return f"" + return f"<{self.__class__.__name__} {int(self)}%{self.max_value}>" def __str__(self): return str(int(self)) From eb1b2e2f9d18bae145fb47819f9b1af8a1a8bc3f Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sat, 3 Dec 2022 22:36:02 +0200 Subject: [PATCH 14/14] Fix sign --- dpath/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/types.py b/dpath/types.py index 210b24a..7bf3d2d 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -21,7 +21,7 @@ def __eq__(self, other): if not isinstance(other, int): return False - if other >= self.max_value or other >= -self.max_value: + if other >= self.max_value or other <= -self.max_value: return False return int(self) == (self.max_value + other) % self.max_value