7th round of migrating integration tests to TestKit (#685)

* Migrate temporal and spatial ITs to TestKit

* Introduce RegExps for skipping TestKit tests

* Add TestKit protocol messages for subtests

PR in TestKit repo:
neo4j-drivers/testkit#435

Author: robsdedude

CHANGELOG.md

@@ -87,6 +87,13 @@
   does not offer the `commit`, `rollback`, `close`, and `closed` methods.  
   Those methods would have caused a hard to interpreted error previously. Hence,
   they have been removed.
+- Deprecated Nodes' and Relationships' `id` property (`int`) in favor of
+  `element_id` (`str`).  
+  This also affects `Graph` objects as `graph.nodes[...]` and
+ `graph.relationships[...]` now prefers strings over integers.
+- `ServerInfo.connection_id` has been deprecated and will be removed in a
+  future release. There is no replacement as this is considered internal
+  information.
 
 
 ## Version 4.4

neo4j/api.py

@@ -308,6 +308,8 @@ def agent(self):
         return self._metadata.get("server")
 
     @property
+    @deprecated("The connection id is considered internal information "
+                "and will no longer be exposed in future versions.")
     def connection_id(self):
         """ Unique identifier for the remote server connection.
         """

neo4j/graph/__init__.py

@@ -31,7 +31,10 @@
 
 from collections.abc import Mapping
 
-from ..meta import deprecated
+from ..meta import (
+    deprecated,
+    deprecation_warn,
+)
 
 
 class Graph:
@@ -261,6 +264,20 @@ def __init__(self, entity_dict):
         self._entity_dict = entity_dict
 
     def __getitem__(self, e_id):
+        # TODO: 6.0 - remove this compatibility shim
+        if isinstance(e_id, (int, float, complex)):
+            deprecation_warn(
+                "Accessing entities by an integer id is deprecated, "
+                "use the new style element_id (str) instead"
+            )
+            if isinstance(e_id, float) and int(e_id) == e_id:
+                # Non-int floats would always fail for legacy IDs
+                e_id = int(e_id)
+            elif isinstance(e_id, complex) and int(e_id.real) == e_id:
+                # complex numbers with imaginary parts or non-integer real
+                # parts would always fail for legacy IDs
+                e_id = int(e_id.real)
+            e_id = str(e_id)
         return self._entity_dict[e_id]
 
     def __len__(self):

neo4j/time/hydration.py

@@ -136,7 +136,7 @@ def dehydrate_datetime(value):
     """ Dehydrator for `datetime` values.
 
     :param value:
-    :type value: datetime
+    :type value: datetime or DateTime
     :return:
     """
 

testkit/build.py

@@ -19,18 +19,22 @@
 
 
 """
-Executed in Go driver container.
+Executed in driver container.
 Responsible for building driver and test backend.
 """
 
 
 import subprocess
+import sys
 
 
 def run(args, env=None):
-    subprocess.run(args, universal_newlines=True, stderr=subprocess.STDOUT,
-                   check=True, env=env)
+    subprocess.run(args, universal_newlines=True, stdout=sys.stdout,
+                   stderr=sys.stderr, check=True, env=env)
 
 
 if __name__ == "__main__":
     run(["python", "setup.py", "build"])
+    run(["python", "-m", "pip", "install", "-U", "pip"])
+    run(["python", "-m", "pip", "install", "-Ur",
+         "testkitbackend/requirements.txt"])

testkitbackend/_async/requests.py

@@ -18,12 +18,15 @@
 
 import json
 from os import path
+import re
+import warnings
 
 import neo4j
 from neo4j._async_compat.util import AsyncUtil
 
 from .. import (
     fromtestkit,
+    test_subtest_skips,
     totestkit,
 )
 from ..exceptions import MarkdAsDriverException
@@ -48,11 +51,38 @@ def load_config():
 SKIPPED_TESTS, FEATURES = load_config()
 
 
+def _get_skip_reason(test_name):
+    for skip_pattern, reason in SKIPPED_TESTS.items():
+        if skip_pattern[0] == skip_pattern[-1] == "'":
+            match = skip_pattern[1:-1] == test_name
+        else:
+            match = re.match(skip_pattern, test_name)
+        if match:
+            return reason
+
+
 async def StartTest(backend, data):
-    if data["testName"] in SKIPPED_TESTS:
-        await backend.send_response("SkipTest", {
-            "reason": SKIPPED_TESTS[data["testName"]]
-        })
+    test_name = data["testName"]
+    reason = _get_skip_reason(test_name)
+    if reason is not None:
+        if reason.startswith("test_subtest_skips."):
+            await backend.send_response("RunSubTests", {})
+        else:
+            await backend.send_response("SkipTest", {"reason": reason})
+    else:
+        await backend.send_response("RunTest", {})
+
+
+async def StartSubTest(backend, data):
+    test_name = data["testName"]
+    subtest_args = data["subtestArguments"]
+    subtest_args.mark_all_as_read(recursive=True)
+    reason = _get_skip_reason(test_name)
+    assert reason and reason.startswith("test_subtest_skips.") or print(reason)
+    func = getattr(test_subtest_skips, reason[19:])
+    reason = func(**subtest_args)
+    if reason is not None:
+        await backend.send_response("SkipTest", {"reason": reason})
     else:
         await backend.send_response("RunTest", {})
 
@@ -412,6 +442,17 @@ async def ResultSingle(backend, data):
     ))
 
 
+async def ResultSingleOptional(backend, data):
+    result = backend.results[data["resultId"]]
+    with warnings.catch_warnings(record=True) as warning_list:
+        record = await result.single(strict=False)
+    if record:
+        record = totestkit.record(record)
+    await backend.send_response("RecordOptional", {
+        "record": record, "warnings": list(map(str, warning_list))
+    })
+
+
 async def ResultPeek(backend, data):
     result = backend.results[data["resultId"]]
     record = await result.peek()

testkitbackend/_sync/requests.py

@@ -18,12 +18,15 @@
 
 import json
 from os import path
+import re
+import warnings
 
 import neo4j
 from neo4j._async_compat.util import Util
 
 from .. import (
     fromtestkit,
+    test_subtest_skips,
     totestkit,
 )
 from ..exceptions import MarkdAsDriverException
@@ -48,11 +51,38 @@ def load_config():
 SKIPPED_TESTS, FEATURES = load_config()
 
 
+def _get_skip_reason(test_name):
+    for skip_pattern, reason in SKIPPED_TESTS.items():
+        if skip_pattern[0] == skip_pattern[-1] == "'":
+            match = skip_pattern[1:-1] == test_name
+        else:
+            match = re.match(skip_pattern, test_name)
+        if match:
+            return reason
+
+
 def StartTest(backend, data):
-    if data["testName"] in SKIPPED_TESTS:
-        backend.send_response("SkipTest", {
-            "reason": SKIPPED_TESTS[data["testName"]]
-        })
+    test_name = data["testName"]
+    reason = _get_skip_reason(test_name)
+    if reason is not None:
+        if reason.startswith("test_subtest_skips."):
+            backend.send_response("RunSubTests", {})
+        else:
+            backend.send_response("SkipTest", {"reason": reason})
+    else:
+        backend.send_response("RunTest", {})
+
+
+def StartSubTest(backend, data):
+    test_name = data["testName"]
+    subtest_args = data["subtestArguments"]
+    subtest_args.mark_all_as_read(recursive=True)
+    reason = _get_skip_reason(test_name)
+    assert reason and reason.startswith("test_subtest_skips.") or print(reason)
+    func = getattr(test_subtest_skips, reason[19:])
+    reason = func(**subtest_args)
+    if reason is not None:
+        backend.send_response("SkipTest", {"reason": reason})
     else:
         backend.send_response("RunTest", {})
 
@@ -412,6 +442,17 @@ def ResultSingle(backend, data):
     ))
 
 
+def ResultSingleOptional(backend, data):
+    result = backend.results[data["resultId"]]
+    with warnings.catch_warnings(record=True) as warning_list:
+        record = result.single(strict=False)
+    if record:
+        record = totestkit.record(record)
+    backend.send_response("RecordOptional", {
+        "record": record, "warnings": list(map(str, warning_list))
+    })
+
+
 def ResultPeek(backend, data):
     result = backend.results[data["resultId"]]
     record = result.peek()

testkitbackend/fromtestkit.py

@@ -16,7 +16,21 @@
 # limitations under the License.
 
 
+from datetime import timedelta
+
+import pytz
+
 from neo4j import Query
+from neo4j.spatial import (
+    CartesianPoint,
+    WGS84Point,
+)
+from neo4j.time import (
+    Date,
+    DateTime,
+    Duration,
+    Time,
+)
 
 
 def to_cypher_and_params(data):
@@ -54,24 +68,81 @@ def to_query_and_params(data):
 def to_param(m):
     """ Converts testkit parameter format to driver (python) parameter
     """
-    value = m["data"]["value"]
+    data = m["data"]
     name = m["name"]
     if name == "CypherNull":
+        if data["value"] is not None:
+            raise ValueError("CypherNull should be None")
         return None
     if name == "CypherString":
-        return str(value)
+        return str(data["value"])
     if name == "CypherBool":
-        return bool(value)
+        return bool(data["value"])
     if name == "CypherInt":
-        return int(value)
+        return int(data["value"])
     if name == "CypherFloat":
-        return float(value)
+        return float(data["value"])
     if name == "CypherString":
-        return str(value)
+        return str(data["value"])
     if name == "CypherBytes":
-        return bytearray([int(byte, 16) for byte in value.split()])
+        return bytearray([int(byte, 16) for byte in data["value"].split()])
     if name == "CypherList":
-        return [to_param(v) for v in value]
+        return [to_param(v) for v in data["value"]]
     if name == "CypherMap":
-        return {k: to_param(value[k]) for k in value}
-    raise Exception("Unknown param type " + name)
+        return {k: to_param(data["value"][k]) for k in data["value"]}
+    if name == "CypherPoint":
+        coords = [data["x"], data["y"]]
+        if data.get("z") is not None:
+            coords.append(data["z"])
+        if data["system"] == "cartesian":
+            return CartesianPoint(coords)
+        if data["system"] == "wgs84":
+            return WGS84Point(coords)
+        raise ValueError("Unknown point system: {}".format(data["system"]))
+    if name == "CypherDate":
+        return Date(data["year"], data["month"], data["day"])
+    if name == "CypherTime":
+        tz = None
+        utc_offset_s = data.get("utc_offset_s")
+        if utc_offset_s is not None:
+            utc_offset_m = utc_offset_s // 60
+            if utc_offset_m * 60 != utc_offset_s:
+                raise ValueError("the used timezone library only supports "
+                                 "UTC offsets by minutes")
+            tz = pytz.FixedOffset(utc_offset_m)
+        return Time(data["hour"], data["minute"], data["second"],
+                    data["nanosecond"], tzinfo=tz)
+    if name == "CypherDateTime":
+        datetime = DateTime(
+            data["year"], data["month"], data["day"],
+            data["hour"], data["minute"], data["second"], data["nanosecond"]
+        )
+        utc_offset_s = data["utc_offset_s"]
+        timezone_id = data["timezone_id"]
+        if timezone_id is not None:
+            utc_offset = timedelta(seconds=utc_offset_s)
+            tz = pytz.timezone(timezone_id)
+            localized_datetime = tz.localize(datetime, is_dst=False)
+            if localized_datetime.utcoffset() == utc_offset:
+                return localized_datetime
+            localized_datetime = tz.localize(datetime, is_dst=True)
+            if localized_datetime.utcoffset() == utc_offset:
+                return localized_datetime
+            raise ValueError(
+                "cannot localize datetime %s to timezone %s with UTC "
+                "offset %s" % (datetime, timezone_id, utc_offset)
+            )
+        elif utc_offset_s is not None:
+            utc_offset_m = utc_offset_s // 60
+            if utc_offset_m * 60 != utc_offset_s:
+                raise ValueError("the used timezone library only supports "
+                                 "UTC offsets by minutes")
+            tz = pytz.FixedOffset(utc_offset_m)
+            return tz.localize(datetime)
+        return datetime
+    if name == "CypherDuration":
+        return Duration(
+            months=data["months"], days=data["days"],
+            seconds=data["seconds"], nanoseconds=data["nanoseconds"]
+        )
+    raise ValueError("Unknown param type " + name)

testkitbackend/requirements.txt

@@ -0,0 +1 @@
+-r ../requirements.txt