Added a stub test to verify that after a leader in a cluster is removed, the driver should be able to retry the tx on 1) writer no longer available error and 2) no writer available error.

Author: Zhen Li

driver/src/test/java/org/neo4j/driver/internal/RoutingDriverBoltKitTest.java

@@ -726,6 +726,35 @@ void shouldRetryWriteTransactionUntilSuccess() throws Exception
         }
     }
 
+    @Test
+    void shouldRetryWriteTransactionUntilSuccessWithWhenLeaderIsRemoved() throws Exception
+    {
+        // This test simulates a router in a cluster that a leader is removed.
+        // The router first returns a RT with a writer inside.
+        // However this writer is killed while the driver is running a tx with it.
+        // As a result, the writer server is removed from RT in the router's second reply.
+        // Finally, the router will return a RT with a reachable writer.
+        StubServer router = StubServer.start( "acquire_endpoints_v3_leader_killed.script", 9001 );
+        StubServer brokenWriter = StubServer.start( "dead_write_server.script", 9004 );
+        StubServer writer = StubServer.start( "write_server.script", 9008 );
+
+        try ( Driver driver = newDriverWithSleeplessClock( "bolt+routing://127.0.0.1:9001" );
+                Session session = driver.session() )
+        {
+            AtomicInteger invocations = new AtomicInteger();
+            List<Record> records = session.writeTransaction( queryWork( "CREATE (n {name:'Bob'})", invocations ) );
+
+            assertEquals( 0, records.size() );
+            assertEquals( 2, invocations.get() );
+        }
+        finally
+        {
+            assertEquals( 0, router.exitStatus() );
+            assertEquals( 0, brokenWriter.exitStatus() );
+            assertEquals( 0, writer.exitStatus() );
+        }
+    }
+
     @Test
     void shouldRetryReadTransactionUntilFailure() throws Exception
     {

driver/src/test/resources/acquire_endpoints_v3_leader_killed.script

@@ -0,0 +1,20 @@
+!: BOLT 3
+!: AUTO RESET
+
+C: HELLO {"scheme": "none", "user_agent": "neo4j-java/dev"}
+S: SUCCESS {"server": "Neo4j/9.9.9", "connection_id": "bolt-123456789"}
+C: RUN "CALL dbms.cluster.routing.getRoutingTable({context})" {"context": {}} {}
+   PULL_ALL
+S: SUCCESS {"fields": ["ttl", "servers"]}
+   RECORD [9223372036854775807, [{"addresses": ["127.0.0.1:9004"],"role": "WRITE"}, {"addresses": ["127.0.0.1:9005","127.0.0.1:9006"], "role": "READ"},{"addresses": ["127.0.0.1:9001"], "role": "ROUTE"}]]
+   SUCCESS {}
+C: RUN "CALL dbms.cluster.routing.getRoutingTable({context})" {"context": {}} {}
+   PULL_ALL
+S: SUCCESS {"fields": ["ttl", "servers"]}
+   RECORD [9223372036854775807, [{"addresses": [],"role": "WRITE"}, {"addresses": ["127.0.0.1:9005","127.0.0.1:9006"], "role": "READ"},{"addresses": ["127.0.0.1:9001"], "role": "ROUTE"}]]
+   SUCCESS {}
+C: RUN "CALL dbms.cluster.routing.getRoutingTable({context})" {"context": {}} {}
+   PULL_ALL
+S: SUCCESS {"fields": ["ttl", "servers"]}
+   RECORD [9223372036854775807, [{"addresses": ["127.0.0.1:9008"],"role": "WRITE"}, {"addresses": ["127.0.0.1:9005","127.0.0.1:9006"], "role": "READ"},{"addresses": ["127.0.0.1:9001"], "role": "ROUTE"}]]
+   SUCCESS {}