Migrate stub tests to testkit

 - remove all migrated stub tests
 - some stub tests (or parts of them) got turned into unit tests
 - implement testkit messages ForcedRoutingTableUpdate and GetRoutingTable
 - send debug log output to testkit to easy test debugging for people running
   the backend in docker or similar

Author: robsdedude

testkitbackend/backend.py

@@ -14,7 +14,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from inspect import getmembers, isfunction
+from inspect import (
+    getmembers,
+    isfunction,
+)
+import io
 from json import loads, dumps
 import logging
 import sys
@@ -28,9 +32,13 @@
 
 import testkitbackend.requests as requests
 
+buffer_handler = logging.StreamHandler(io.StringIO())
+buffer_handler.setLevel(logging.DEBUG)
+
 handler = logging.StreamHandler(sys.stdout)
 handler.setLevel(logging.DEBUG)
 logging.getLogger("neo4j").addHandler(handler)
+logging.getLogger("neo4j").addHandler(buffer_handler)
 logging.getLogger("neo4j").setLevel(logging.DEBUG)
 
 log = logging.getLogger("testkitbackend")
@@ -165,6 +173,14 @@ def _process(self, request):
     def send_response(self, name, data):
         """ Sends a response to backend.
         """
+        buffer_handler.acquire()
+        log_output = buffer_handler.stream.getvalue()
+        buffer_handler.stream.truncate(0)
+        buffer_handler.stream.seek(0)
+        buffer_handler.release()
+        if not log_output.endswith("\n"):
+            log_output += "\n"
+        self._wr.write(log_output.encode("utf-8"))
         response = {"name": name, "data": data}
         response = dumps(response)
         log.info(">>> " + name + dumps(data))

testkitbackend/requests.py

@@ -23,8 +23,14 @@
 from testkitbackend.fromtestkit import to_meta_and_timeout
 
 
-with open(path.join(path.dirname(__file__), "skipped_tests.json"), "r") as fd:
-    SKIPPED_TESTS = json.load(fd)
+def load_config():
+    with open(path.join(path.dirname(__file__), "test_config.json"), "r") as fd:
+        config = json.load(fd)
+    return (config["skips"],
+            [k for k, v in config["features"].items() if v is True])
+
+
+SKIPPED_TESTS, FEATURES = load_config()
 
 
 def StartTest(backend, data):
@@ -35,6 +41,10 @@ def StartTest(backend, data):
         backend.send_response("RunTest", {})
 
 
+def GetFeatures(backend, data):
+    backend.send_response("FeatureList", {"features": FEATURES})
+
+
 def NewDriver(backend, data):
     auth_token = data["authorizationToken"]["data"]
     data["authorizationToken"].mark_item_as_read_if_equals(
@@ -294,7 +304,8 @@ def ResultConsume(backend, data):
         "serverInfo": {
             "protocolVersion":
                 ".".join(map(str, summary.server.protocol_version)),
-            "agent": summary.server.agent
+            "agent": summary.server.agent,
+            # "address": ":".join(map(str, summary.server.address))
         }
     })
 
@@ -310,3 +321,30 @@ def RetryableNegative(backend, data):
     session_tracker = backend.sessions[key]
     session_tracker.state = '-'
     session_tracker.error_id = data.get('errorId', '')
+
+
+def ForcedRoutingTableUpdate(backend, data):
+    driver_id = data["driverId"]
+    driver = backend.drivers[driver_id]
+    database = data["database"]
+    bookmarks = data["bookmarks"]
+    with driver._pool.refresh_lock:
+        driver._pool.create_routing_table(database)
+        driver._pool.update_routing_table(database=database,
+                                          bookmarks=bookmarks)
+    backend.send_response("Driver", {"id": driver_id})
+
+
+def GetRoutingTable(backend, data):
+    driver_id = data["driverId"]
+    database = data["database"]
+    driver = backend.drivers[driver_id]
+    routing_table = driver._pool.routing_tables[database]
+    response_data = {
+        "database": routing_table.database,
+        "ttl": routing_table.ttl,
+    }
+    for role in ("routers", "readers", "writers"):
+        addresses = routing_table.__getattribute__(role)
+        response_data[role] = list(map(str, addresses))
+    backend.send_response("RoutingTable", response_data)

testkitbackend/skipped_tests.json

@@ -0,0 +1,37 @@
+{
+  "skips": {
+    "stub.routing.Routing.test_should_retry_write_until_success_with_leader_change_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.RoutingV3.test_should_retry_write_until_success_with_leader_change_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.RoutingV4.test_should_retry_write_until_success_with_leader_change_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.Routing.test_should_retry_write_until_success_with_leader_shutdown_during_tx_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.RoutingV3.test_should_retry_write_until_success_with_leader_shutdown_during_tx_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.RoutingV4.test_should_retry_write_until_success_with_leader_shutdown_during_tx_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.Routing.test_should_successfully_acquire_rt_when_router_ip_changes":
+      "Test makes assumptions about how verify_connectivity is implemented",
+    "stub.routing.RoutingV3.test_should_successfully_acquire_rt_when_router_ip_changes":
+      "Test makes assumptions about how verify_connectivity is implemented",
+    "stub.routing.RoutingV4.test_should_successfully_acquire_rt_when_router_ip_changes":
+      "Test makes assumptions about how verify_connectivity is implemented",
+    "stub.retry.TestRetryClustering.test_retry_ForbiddenOnReadOnlyDatabase_ChangingWriter":
+      "Test makes assumptions about how verify_connectivity is implemented",
+    "stub.authorization.AuthorizationTests.test_should_retry_on_auth_expired_on_begin_using_tx_function":
+      "Flaky: test requires the driver to contact servers in a specific order",
+    "stub.authorization.AuthorizationTestsV3.test_should_retry_on_auth_expired_on_begin_using_tx_function":
+      "Flaky: test requires the driver to contact servers in a specific order",
+    "stub.authorization.AuthorizationTestsV4.test_should_retry_on_auth_expired_on_begin_using_tx_function":
+      "Flaky: test requires the driver to contact servers in a specific order"
+  },
+  "features": {
+    "AuthorizationExpiredTreatment": true,
+    "Optimization:ImplicitDefaultArguments": true,
+    "Optimization:MinimalResets": "Driver resets some clean connections when put back into pool",
+    "Optimization:ConnectionReuse": true,
+    "Optimization:PullPipelining": true
+  }
+}

testkitbackend/test_config.json

@@ -36,7 +36,9 @@ class ReadWorkload(object):
     def setup_class(cls):
         cls.server = server = RemoteGraphDatabaseServer()
         server.start()
-        cls.driver = GraphDatabase.driver(server.server_uri, auth=server.auth_token, encrypted=server.encrypted)
+        cls.driver = GraphDatabase.driver(server.server_uri,
+                                          auth=server.auth_token,
+                                          encrypted=server.encrypted)
 
     @classmethod
     def teardown_class(cls):

tests/performance/test_results.py

@@ -19,26 +19,17 @@
 # limitations under the License.
 
 
-from test.integration.tools import IntegrationTestCase
-
-from os import makedirs, remove
-from os.path import basename, dirname, join as path_join, realpath, isfile, expanduser
-import platform
 from unittest import TestCase, SkipTest
-from shutil import copyfile
-from sys import exit, stderr
 
 try:
     from urllib.request import urlretrieve
 except ImportError:
     from urllib import urlretrieve
 
-from boltkit.controller import WindowsController, UnixController
-
 from neo4j import GraphDatabase
 from neo4j.exceptions import AuthError
 
-from test.env import NEO4J_USER, NEO4J_PASSWORD, NEO4J_SERVER_URI
+from tests.env import NEO4J_USER, NEO4J_PASSWORD, NEO4J_SERVER_URI
 
 
 def is_listening(address):

tests/performance/tools.py

@@ -3,4 +3,5 @@ coverage
 pytest
 pytest-benchmark
 pytest-cov
+pytest-mock
 teamcity-messages