testMockFailToSendNoConnectRule should wait for connection close to bubble up and disconnect the node

elastic#25521 changed channel closing to be handled async on anything but transport stop. This means it may take a while before
calling `connection.close()` and the node being removed from the `connectedNodes` list (but the connection is immediately unusuable).

Fixes elastic#25686

Author: bleskes

test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java

@@ -30,7 +30,6 @@
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -80,7 +79,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collector;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
@@ -1418,7 +1416,7 @@ public String executor() {
         assertThat(version0Response.value1, equalTo(1));
     }
 
-    public void testMockFailToSendNoConnectRule() throws IOException {
+    public void testMockFailToSendNoConnectRule() throws Exception {
         serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC,
             (request, channel) -> {
                 assertThat("moshe", equalTo(request.message));
@@ -1461,6 +1459,10 @@ public void handleException(TransportException exp) {
             assertThat(((ConnectTransportException)cause).node(), equalTo(nodeA));
         }
 
+        // wait for the transport to process the sending failure and disconnect from node
+        assertBusy(() -> assertFalse(serviceB.nodeConnected(nodeA)));
+
+        // now try to connect again and see that it fails
         try {
             serviceB.connectToNode(nodeA);
             fail("exception should be thrown");