From 6b6981f42c0a144a09fac999ab628d55c8fc0266 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 6 Jul 2018 11:45:55 -0600 Subject: [PATCH 01/29] Work on connection manager --- .../transport/ConnectionManager.java | 150 ++++++++++++++ .../elasticsearch/transport/TcpTransport.java | 188 ++++-------------- .../elasticsearch/transport/Transport.java | 38 +--- .../TransportConnectionListener.java | 12 ++ .../transport/TransportService.java | 10 +- .../transport/FailAndRetryMockTransport.java | 144 ++++++-------- .../TransportClientNodesServiceTests.java | 17 +- .../cluster/NodeConnectionsServiceTests.java | 92 +++++---- .../discovery/DiscoveryDisruptionIT.java | 2 +- .../discovery/zen/UnicastZenPingTests.java | 9 +- .../RemoteClusterConnectionTests.java | 38 ++-- .../transport/TcpTransportTests.java | 4 +- .../test/transport/CapturingTransport.java | 29 +-- .../test/transport/MockTransportService.java | 111 +---------- 14 files changed, 374 insertions(+), 470 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/transport/ConnectionManager.java diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java new file mode 100644 index 0000000000000..daa22d9c0c2ef --- /dev/null +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -0,0 +1,150 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.util.concurrent.KeyedLock; +import org.elasticsearch.core.internal.io.IOUtils; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; + +public class ConnectionManager { + + private final ConcurrentMap connectedNodes = newConcurrentMap(); + private final KeyedLock connectionLock = new KeyedLock<>(); + private final AtomicBoolean isClosed = new AtomicBoolean(false); + private final Logger logger; + private final DelegatingNodeConnectionListener connectionListener = new DelegatingNodeConnectionListener(); + + public ConnectionManager(Logger logger) { + this.logger = logger; + } + + public void registerListener(TransportConnectionListener.NodeConnectionListener listener) { + this.connectionListener.listeners.add(listener); + } + + public void connectToNode(Transport transport, DiscoveryNode node, ConnectionProfile connectionProfile, + CheckedBiConsumer connectionValidator) + throws ConnectTransportException { + if (node == null) { + throw new ConnectTransportException(null, "can't connect to a null node"); + } + ensureOpen(); + try (Releasable ignored = connectionLock.acquire(node.getId())) { + Transport.Connection connection = connectedNodes.get(node); + if (connection != null) { + return; + } + boolean success = false; + try { + connection = transport.openConnection(node, connectionProfile); + connectionValidator.accept(connection, connectionProfile); + // we acquire a connection lock, so no way there is an existing connection + connectedNodes.put(node, connection); + if (logger.isDebugEnabled()) { + logger.debug("connected to node [{}]", node); + } + try { + connectionListener.onNodeConnected(node); + } finally { + if (connection.isClosed()) { + // we got closed concurrently due to a disconnect or some other event on the channel. + // the close callback will close the NodeChannel instance first and then try to remove + // the connection from the connected nodes. It will NOT acquire the connectionLock for + // the node to prevent any blocking calls on network threads. Yet, we still establish a happens + // before relationship to the connectedNodes.put since we check if we can remove the + // (DiscoveryNode, NodeChannels) tuple from the map after we closed. Here we check if it's closed an if so we + // try to remove it first either way one of the two wins even if the callback has run before we even added the + // tuple to the map since in that case we remove it here again + if (connectedNodes.remove(node, connection)) { + connectionListener.onNodeDisconnected(node); + } + throw new NodeNotConnectedException(node, "connection concurrently closed"); + } + } + success = true; + } catch (ConnectTransportException e) { + throw e; + } catch (Exception e) { + throw new ConnectTransportException(node, "general node connection failure", e); + } finally { + if (success == false) { // close the connection if there is a failure + logger.trace(() -> new ParameterizedMessage("failed to connect to [{}], cleaning dangling connections", node)); + IOUtils.closeWhileHandlingException(connection); + } + } + } + } + + public Transport.Connection getConnection(DiscoveryNode node) { + Transport.Connection connection = connectedNodes.get(node); + if (connection == null) { + throw new NodeNotConnectedException(node, "Node not connected"); + } + return connection; + } + + public boolean nodeConnected(DiscoveryNode node) { + return connectedNodes.containsKey(node); + } + + public void disconnectFromNode(DiscoveryNode node) { + // TODO: Do we need to lock here? + Transport.Connection nodeChannels = connectedNodes.remove(node); + if (nodeChannels != null) { // if we found it and removed it we close and notify + IOUtils.closeWhileHandlingException(nodeChannels, () -> connectionListener.onNodeDisconnected(node)); + } + } + + private void ensureOpen() { + if (isClosed.get()) { + throw new IllegalStateException("connection manager is closed"); + } + } + + private static final class DelegatingNodeConnectionListener implements TransportConnectionListener.NodeConnectionListener { + + private final List listeners = new CopyOnWriteArrayList<>(); + + @Override + public void onNodeDisconnected(DiscoveryNode key) { + for (TransportConnectionListener.NodeConnectionListener listener : listeners) { + listener.onNodeDisconnected(key); + } + } + + @Override + public void onNodeConnected(DiscoveryNode node) { + for (TransportConnectionListener.NodeConnectionListener listener : listeners) { + listener.onNodeConnected(node); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index b2294ce589325..214a0d662739c 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -29,7 +29,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Booleans; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -48,7 +47,6 @@ import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.network.CloseableChannel; @@ -66,7 +64,6 @@ import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.common.util.concurrent.KeyedLock; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -90,7 +87,6 @@ import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -209,12 +205,9 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements private final DelegatingTransportConnectionListener transportListener = new DelegatingTransportConnectionListener(); private final ConcurrentMap profileBoundAddresses = newConcurrentMap(); - // node id to actual channel - private final ConcurrentMap connectedNodes = newConcurrentMap(); private final Map> serverChannels = newConcurrentMap(); private final Set acceptedChannels = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final KeyedLock connectionLock = new KeyedLock<>(); private final NamedWriteableRegistry namedWriteableRegistry; // this lock is here to make sure we close this transport and disconnect all the client nodes @@ -371,30 +364,30 @@ public ScheduledPing() { @Override protected void doRunInLifecycle() throws Exception { - for (Map.Entry entry : connectedNodes.entrySet()) { - DiscoveryNode node = entry.getKey(); - NodeChannels channels = entry.getValue(); - for (TcpChannel channel : channels.getChannels()) { - internalSendMessage(channel, pingHeader, new SendMetricListener(pingHeader.length()) { - @Override - protected void innerInnerOnResponse(Void v) { - successfulPings.inc(); - } - - @Override - protected void innerOnFailure(Exception e) { - if (channel.isOpen()) { - logger.debug(() -> new ParameterizedMessage("[{}] failed to send ping transport message", node), e); - failedPings.inc(); - } else { - logger.trace(() -> - new ParameterizedMessage("[{}] failed to send ping transport message (channel closed)", node), e); - } - - } - }); - } - } +// for (Map.Entry entry : connectedNodes.entrySet()) { +// DiscoveryNode node = entry.getKey(); +// NodeChannels channels = entry.getValue(); +// for (TcpChannel channel : channels.getChannels()) { +// internalSendMessage(channel, pingHeader, new SendMetricListener(pingHeader.length()) { +// @Override +// protected void innerInnerOnResponse(Void v) { +// successfulPings.inc(); +// } +// +// @Override +// protected void innerOnFailure(Exception e) { +// if (channel.isOpen()) { +// logger.debug(() -> new ParameterizedMessage("[{}] failed to send ping transport message", node), e); +// failedPings.inc(); +// } else { +// logger.trace(() -> +// new ParameterizedMessage("[{}] failed to send ping transport message (channel closed)", node), e); +// } +// +// } +// }); +// } +// } } public long getSuccessfulPings() { @@ -514,76 +507,12 @@ public void sendRequest(long requestId, String action, TransportRequest request, sendRequestToChannel(this.node, channel, requestId, action, request, options, getVersion(), (byte) 0); } - boolean isClosed() { + @Override + public boolean isClosed() { return closed.get(); } } - @Override - public boolean nodeConnected(DiscoveryNode node) { - return connectedNodes.containsKey(node); - } - - @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - connectionProfile = resolveConnectionProfile(connectionProfile); - if (node == null) { - throw new ConnectTransportException(null, "can't connect to a null node"); - } - closeLock.readLock().lock(); // ensure we don't open connections while we are closing - try { - ensureOpen(); - try (Releasable ignored = connectionLock.acquire(node.getId())) { - NodeChannels nodeChannels = connectedNodes.get(node); - if (nodeChannels != null) { - return; - } - boolean success = false; - try { - nodeChannels = openConnection(node, connectionProfile); - connectionValidator.accept(nodeChannels, connectionProfile); - // we acquire a connection lock, so no way there is an existing connection - connectedNodes.put(node, nodeChannels); - if (logger.isDebugEnabled()) { - logger.debug("connected to node [{}]", node); - } - try { - transportListener.onNodeConnected(node); - } finally { - if (nodeChannels.isClosed()) { - // we got closed concurrently due to a disconnect or some other event on the channel. - // the close callback will close the NodeChannel instance first and then try to remove - // the connection from the connected nodes. It will NOT acquire the connectionLock for - // the node to prevent any blocking calls on network threads. Yet, we still establish a happens - // before relationship to the connectedNodes.put since we check if we can remove the - // (DiscoveryNode, NodeChannels) tuple from the map after we closed. Here we check if it's closed an if so we - // try to remove it first either way one of the two wins even if the callback has run before we even added the - // tuple to the map since in that case we remove it here again - if (connectedNodes.remove(node, nodeChannels)) { - transportListener.onNodeDisconnected(node); - } - throw new NodeNotConnectedException(node, "connection concurrently closed"); - } - } - success = true; - } catch (ConnectTransportException e) { - throw e; - } catch (Exception e) { - throw new ConnectTransportException(node, "general node connection failure", e); - } finally { - if (success == false) { // close the connection if there is a failure - logger.trace(() -> new ParameterizedMessage("failed to connect to [{}], cleaning dangling connections", node)); - IOUtils.closeWhileHandlingException(nodeChannels); - } - } - } - } finally { - closeLock.readLock().unlock(); - } - } - /** * takes a {@link ConnectionProfile} that have been passed as a parameter to the public methods * and resolves it to a fully specified (i.e., no nulls) profile @@ -612,7 +541,7 @@ protected ConnectionProfile resolveConnectionProfile(ConnectionProfile connectio } @Override - public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) { + public NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) { if (node == null) { throw new ConnectTransportException(null, "can't open connection to a null node"); } @@ -672,7 +601,8 @@ public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile c // we only need to disconnect from the nodes once since all other channels // will also try to run this we protect it from running multiple times. if (runOnce.compareAndSet(false, true)) { - disconnectFromNodeCloseAndNotify(node, finalNodeChannels); + // TODO: Have lost the removal from connection manager + IOUtils.closeWhileHandlingException(finalNodeChannels); } }; @@ -699,46 +629,6 @@ public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile c } } - private void disconnectFromNodeCloseAndNotify(DiscoveryNode node, NodeChannels nodeChannels) { - assert nodeChannels != null : "nodeChannels must not be null"; - try { - IOUtils.closeWhileHandlingException(nodeChannels); - } finally { - if (closeLock.readLock().tryLock()) { - try { - if (connectedNodes.remove(node, nodeChannels)) { - transportListener.onNodeDisconnected(node); - } - } finally { - closeLock.readLock().unlock(); - } - } - } - } - - @Override - public NodeChannels getConnection(DiscoveryNode node) { - NodeChannels nodeChannels = connectedNodes.get(node); - if (nodeChannels == null) { - throw new NodeNotConnectedException(node, "Node not connected"); - } - return nodeChannels; - } - - @Override - public void disconnectFromNode(DiscoveryNode node) { - closeLock.readLock().lock(); - NodeChannels nodeChannels = null; - try (Releasable ignored = connectionLock.acquire(node.getId())) { - nodeChannels = connectedNodes.remove(node); - } finally { - closeLock.readLock().unlock(); - if (nodeChannels != null) { // if we found it and removed it we close and notify - IOUtils.closeWhileHandlingException(nodeChannels, () -> transportListener.onNodeDisconnected(node)); - } - } - } - protected Version getCurrentVersion() { // this is just for tests to mock stuff like the nodes version - tests can override this internally return Version.CURRENT; @@ -986,16 +876,16 @@ protected final void doStop() { // we are holding a write lock so nobody modifies the connectedNodes / openConnections map - it's safe to first close // all instances and then clear them maps - Iterator> iterator = connectedNodes.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry next = iterator.next(); - try { - IOUtils.closeWhileHandlingException(next.getValue()); - transportListener.onNodeDisconnected(next.getKey()); - } finally { - iterator.remove(); - } - } +// Iterator> iterator = connectedNodes.entrySet().iterator(); +// while (iterator.hasNext()) { +// Map.Entry next = iterator.next(); +// try { +// IOUtils.closeWhileHandlingException(next.getValue()); +// transportListener.onNodeDisconnected(next.getKey()); +// } finally { +// iterator.remove(); +// } +// } stopInternal(); } finally { closeLock.writeLock().unlock(); diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java index 74235479657bf..e5e4262106d34 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transport.java +++ b/server/src/main/java/org/elasticsearch/transport/Transport.java @@ -85,23 +85,6 @@ public interface Transport extends LifecycleComponent { */ TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException; - /** - * Returns {@code true} if the node is connected. - */ - boolean nodeConnected(DiscoveryNode node); - - /** - * Connects to a node with the given connection profile. If the node is already connected this method has no effect. - * Once a successful is established, it can be validated before being exposed. - */ - void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) throws ConnectTransportException; - - /** - * Disconnected from the given node, if not connected, will do nothing. - */ - void disconnectFromNode(DiscoveryNode node); - /** * Returns a list of all local adresses for this transport */ @@ -112,23 +95,10 @@ default CircuitBreaker getInFlightRequestBreaker() { } /** - * Returns a connection for the given node if the node is connected. - * Connections returned from this method must not be closed. The lifecycle of this connection is maintained by the Transport - * implementation. - * - * @throws NodeNotConnectedException if the node is not connected - * @see #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer) - */ - Connection getConnection(DiscoveryNode node); - - /** - * Opens a new connection to the given node and returns it. In contrast to - * {@link #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer)} the returned connection is not managed by + * Opens a new connection to the given node and returns it. The returned connection is not managed by * the transport implementation. This connection must be closed once it's not needed anymore. - * This connection type can be used to execute a handshake between two nodes before the node will be published via - * {@link #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer)}. */ - Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException; + Connection openConnection(DiscoveryNode node, ConnectionProfile profile); TransportStats getStats(); @@ -154,6 +124,10 @@ interface Connection extends Closeable { void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException; + default boolean isClosed() { + return false; + } + /** * Returns the version of the node this connection was established with. */ diff --git a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java index 0ee2ed5828d44..5533fa07a814c 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java @@ -28,6 +28,18 @@ */ public interface TransportConnectionListener { + interface NodeConnectionListener { + /** + * Called once a node connection is opened and registered. + */ + void onNodeConnected(DiscoveryNode node); + + /** + * Called once a node connection is closed and unregistered. + */ + void onNodeDisconnected(DiscoveryNode node); + } + /** * Called once a request is received * @param requestId the internal request ID diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 032258101f730..a13ddf58314ab 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -87,6 +87,7 @@ public class TransportService extends AbstractLifecycleComponent implements Tran private final boolean connectToRemoteCluster; private final Transport.ResponseHandlers responseHandlers; private final TransportInterceptor interceptor; + private final ConnectionManager connectionManager; // An LRU (don't really care about concurrency here) that holds the latest timed out requests so if they // do show up, we can print more descriptive information about them @@ -152,6 +153,7 @@ public TransportService(Settings settings, Transport transport, ThreadPool threa this.transport = transport; this.threadPool = threadPool; this.localNodeFactory = localNodeFactory; + this.connectionManager = new ConnectionManager(logger); this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings); setTracerLogInclude(TRACE_LOG_INCLUDE_SETTING.get(settings)); setTracerLogExclude(TRACE_LOG_EXCLUDE_SETTING.get(settings)); @@ -305,7 +307,7 @@ public List getLocalAddresses() { * Returns true iff the given node is already connected. */ public boolean nodeConnected(DiscoveryNode node) { - return isLocalNode(node) || transport.nodeConnected(node); + return isLocalNode(node) || connectionManager.nodeConnected(node); } /** @@ -327,7 +329,7 @@ public void connectToNode(final DiscoveryNode node, ConnectionProfile connection if (isLocalNode(node)) { return; } - transport.connectToNode(node, connectionProfile, (newConnection, actualProfile) -> { + connectionManager.connectToNode(transport, node, connectionProfile, (newConnection, actualProfile) -> { // We don't validate cluster names to allow for CCS connections. final DiscoveryNode remote = handshake(newConnection, actualProfile.getHandshakeTimeout().millis(), cn -> true).discoveryNode; if (validateConnections && node.equals(remote) == false) { @@ -462,7 +464,7 @@ public void disconnectFromNode(DiscoveryNode node) { if (isLocalNode(node)) { return; } - transport.disconnectFromNode(node); + connectionManager.disconnectFromNode(node); } public void addConnectionListener(TransportConnectionListener listener) { @@ -533,7 +535,7 @@ public Transport.Connection getConnection(DiscoveryNode node) { if (isLocalNode(node)) { return localNodeConnection; } else { - return transport.getConnection(node); + return connectionManager.getConnection(node); } } diff --git a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java index 669a678b77e31..9ae722efcf457 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java +++ b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java @@ -77,69 +77,70 @@ abstract class FailAndRetryMockTransport imp protected abstract ClusterState getMockClusterState(DiscoveryNode node); - @Override - public Connection getConnection(DiscoveryNode node) { - return new Connection() { - @Override - public DiscoveryNode getNode() { - return node; - } - - @Override - public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws IOException, TransportException { - - //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info - if (connectMode) { - if (TransportLivenessAction.NAME.equals(action)) { - TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); - transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.CLUSTER_NAME_SETTING. - getDefault(Settings.EMPTY), - node)); - } else if (ClusterStateAction.NAME.equals(action)) { - TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); - ClusterState clusterState = getMockClusterState(node); - transportResponseHandler.handleResponse(new ClusterStateResponse(clusterName, clusterState, 0L)); - } else { - throw new UnsupportedOperationException("Mock transport does not understand action " + action); - } - return; - } - - //once nodes are connected we'll just return errors for each sendRequest call - triedNodes.add(node); - - if (random.nextInt(100) > 10) { - connectTransportExceptions.incrementAndGet(); - throw new ConnectTransportException(node, "node not available"); - } else { - if (random.nextBoolean()) { - failures.incrementAndGet(); - //throw whatever exception that is not a subclass of ConnectTransportException - throw new IllegalStateException(); - } else { - TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); - if (random.nextBoolean()) { - successes.incrementAndGet(); - transportResponseHandler.handleResponse(newResponse()); - } else { - failures.incrementAndGet(); - transportResponseHandler.handleException(new TransportException("transport exception")); - } - } - } - } - - @Override - public void close() throws IOException { - - } - }; - } - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { - return getConnection(node); +// @Override +// public Connection getConnection(DiscoveryNode node) { +// return new Connection() { +// @Override +// public DiscoveryNode getNode() { +// return node; +// } +// +// @Override +// public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) +// throws IOException, TransportException { +// +// //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info +// if (connectMode) { +// if (TransportLivenessAction.NAME.equals(action)) { +// TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); +// transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.CLUSTER_NAME_SETTING. +// getDefault(Settings.EMPTY), +// node)); +// } else if (ClusterStateAction.NAME.equals(action)) { +// TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); +// ClusterState clusterState = getMockClusterState(node); +// transportResponseHandler.handleResponse(new ClusterStateResponse(clusterName, clusterState, 0L)); +// } else { +// throw new UnsupportedOperationException("Mock transport does not understand action " + action); +// } +// return; +// } +// +// //once nodes are connected we'll just return errors for each sendRequest call +// triedNodes.add(node); +// +// if (random.nextInt(100) > 10) { +// connectTransportExceptions.incrementAndGet(); +// throw new ConnectTransportException(node, "node not available"); +// } else { +// if (random.nextBoolean()) { +// failures.incrementAndGet(); +// //throw whatever exception that is not a subclass of ConnectTransportException +// throw new IllegalStateException(); +// } else { +// TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); +// if (random.nextBoolean()) { +// successes.incrementAndGet(); +// transportResponseHandler.handleResponse(newResponse()); +// } else { +// failures.incrementAndGet(); +// transportResponseHandler.handleException(new TransportException("transport exception")); +// } +// } +// } +// } +// +// @Override +// public void close() { +// +// } +// }; +// } + + @Override + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { +// return getConnection(node); + return null; } protected abstract Response newResponse(); @@ -175,23 +176,6 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi throw new UnsupportedOperationException(); } - @Override - public boolean nodeConnected(DiscoveryNode node) { - return false; - } - - @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - - } - - @Override - public void disconnectFromNode(DiscoveryNode node) { - - } - @Override public Lifecycle.State lifecycleState() { return null; diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 2beaed1e106e4..d4686e56adeb7 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -361,18 +361,18 @@ public void testSniffNodesSamplerClosesConnections() throws Exception { clientService.addDelegate(remoteService, new MockTransportService.DelegateTransport(clientService.original()) { @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { MockConnection connection = new MockConnection(super.openConnection(node, profile)); establishedConnections.add(connection); return connection; } - @Override - public Connection getConnection(DiscoveryNode node) { - MockConnection connection = new MockConnection(super.getConnection(node)); - reusedConnections.add(connection); - return connection; - } +// @Override +// public Connection getConnection(DiscoveryNode node) { +// MockConnection connection = new MockConnection(super.getConnection(node)); +// reusedConnections.add(connection); +// return connection; +// } }); clientService.start(); @@ -455,7 +455,8 @@ public void close() throws IOException { } } - boolean isClosed() { + @Override + public boolean isClosed() { return closed.get(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index eb93dad5db8a0..b55eb07547aeb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -70,7 +70,7 @@ private List generateNodes() { for (int i = randomIntBetween(20, 50); i > 0; i--) { Set roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))); nodes.add(new DiscoveryNode("node_" + i, "" + i, buildNewFakeTransportAddress(), Collections.emptyMap(), - roles, Version.CURRENT)); + roles, Version.CURRENT)); } return nodes; } @@ -121,7 +121,7 @@ public void testReconnect() { for (int i = 0; i < 3; i++) { // simulate disconnects for (DiscoveryNode node : randomSubsetOf(nodes)) { - transport.disconnectFromNode(node); +// transport.disconnectFromNode(node); } service.new ConnectionChecker().run(); } @@ -175,7 +175,8 @@ final class MockTransport implements Transport { Set connectedNodes = ConcurrentCollections.newConcurrentSet(); volatile boolean randomConnectionExceptions = false; private ResponseHandlers responseHandlers = new ResponseHandlers(); - private TransportConnectionListener listener = new TransportConnectionListener() {}; + private TransportConnectionListener listener = new TransportConnectionListener() { + }; @Override public void registerRequestHandler(RequestHandlerRegistry reg) { @@ -211,33 +212,54 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi return new TransportAddress[0]; } - @Override - public boolean nodeConnected(DiscoveryNode node) { - return connectedNodes.contains(node); - } - - @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - if (connectionProfile == null) { - if (connectedNodes.contains(node) == false && randomConnectionExceptions && randomBoolean()) { - throw new ConnectTransportException(node, "simulated"); - } - connectedNodes.add(node); - listener.onNodeConnected(node); - } - } - - @Override - public void disconnectFromNode(DiscoveryNode node) { - connectedNodes.remove(node); - listener.onNodeDisconnected(node); - } +// @Override +// public boolean nodeConnected(DiscoveryNode node) { +// return connectedNodes.contains(node); +// } + +// @Override +// public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, +// CheckedBiConsumer connectionValidator) +// throws ConnectTransportException { +// if (connectionProfile == null) { +// if (connectedNodes.contains(node) == false && randomConnectionExceptions && randomBoolean()) { +// throw new ConnectTransportException(node, "simulated"); +// } +// connectedNodes.add(node); +// listener.onNodeConnected(node); +// } +// } +// +// @Override +// public void disconnectFromNode(DiscoveryNode node) { +// connectedNodes.remove(node); +// listener.onNodeDisconnected(node); +// } + +// @Override +// public Connection getConnection(DiscoveryNode node) { +// return new Connection() { +// @Override +// public DiscoveryNode getNode() { +// return node; +// } +// +// @Override +// public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) +// throws TransportException { +// +// } +// +// @Override +// public void close() { +// +// } +// }; +// } @Override - public Connection getConnection(DiscoveryNode node) { - return new Connection() { + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { + Connection connection = new Connection() { @Override public DiscoveryNode getNode() { return node; @@ -254,11 +276,6 @@ public void close() { } }; - } - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - Connection connection = getConnection(node); listener.onConnectionOpened(connection); return connection; } @@ -282,13 +299,16 @@ public void removeLifecycleListener(LifecycleListener listener) { } @Override - public void start() {} + public void start() { + } @Override - public void stop() {} + public void stop() { + } @Override - public void close() {} + public void close() { + } @Override public TransportStats getStats() { diff --git a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java index f688291237813..71ed9100f46c0 100644 --- a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java @@ -187,7 +187,7 @@ protected void sendRequest(Transport.Connection connection, long requestId, Stri } @Override - public Transport.Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { + public Transport.Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { return super.openConnection(node, profile); } diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java index eef926a1e1238..95ad5e900485c 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java @@ -152,14 +152,7 @@ public void testSimplePings() throws IOException, InterruptedException, Executio new NoneCircuitBreakerService(), new NamedWriteableRegistry(Collections.emptyList()), networkService, - v) { - @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - throw new AssertionError("zen pings should never connect to node (got [" + node + "])"); - } - }; + v); NetworkHandle handleA = startServices(settings, threadPool, "UZP_A", Version.CURRENT, supplier); closeables.push(handleA.transportService); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 5522550f76c7a..5e63356969fe8 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -441,13 +441,13 @@ public void close() throws IOException { } }; service.addDelegate(seedNode.getAddress(), new MockTransportService.DelegateTransport(service.getOriginalTransport()) { - @Override - public Connection getConnection(DiscoveryNode node) { - if (node == seedNode) { - return seedConnection; - } - return super.getConnection(node); - } +// @Override +// public Connection getConnection(DiscoveryNode node) { +// if (node == seedNode) { +// return seedConnection; +// } +// return super.getConnection(node); +// } }); service.start(); service.acceptIncomingRequests(); @@ -1289,18 +1289,18 @@ public void close() { } }; service.addDelegate(connectedNode.getAddress(), new MockTransportService.DelegateTransport(service.getOriginalTransport()) { - @Override - public Connection getConnection(DiscoveryNode node) { - if (node == connectedNode) { - return seedConnection; - } - return super.getConnection(node); - } - - @Override - public boolean nodeConnected(DiscoveryNode node) { - return node.equals(connectedNode); - } +// @Override +// public Connection getConnection(DiscoveryNode node) { +// if (node == connectedNode) { +// return seedConnection; +// } +// return super.getConnection(node); +// } + +// @Override +// public boolean nodeConnected(DiscoveryNode node) { +// return node.equals(connectedNode); +// } }); service.start(); service.acceptIncomingRequests(); diff --git a/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java b/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java index d16300bf266d6..8474b9947129e 100644 --- a/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java @@ -198,7 +198,7 @@ protected void stopInternal() { } @Override - public NodeChannels getConnection(DiscoveryNode node) { + public NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) { int numConnections = MockTcpTransport.LIGHT_PROFILE.getNumConnections(); ArrayList fakeChannels = new ArrayList<>(numConnections); for (int i = 0; i < numConnections; ++i) { @@ -209,7 +209,7 @@ public NodeChannels getConnection(DiscoveryNode node) { }; DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT); - Transport.Connection connection = transport.getConnection(node); + Transport.Connection connection = transport.openConnection(node, null); connection.sendRequest(42, "foobar", request, TransportRequestOptions.EMPTY); BytesReference reference = messageCaptor.get(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java index ffdf79c0636b2..0985b338f958f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java @@ -195,7 +195,7 @@ public void handleError(final long requestId, final TransportException e) { } @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { return new Connection() { @Override public DiscoveryNode getNode() { @@ -204,13 +204,13 @@ public DiscoveryNode getNode() { @Override public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws IOException, TransportException { + throws TransportException { requests.put(requestId, Tuple.tuple(node, action)); capturedRequests.add(new CapturedRequest(node, requestId, action, request)); } @Override - public void close() throws IOException { + public void close() { } }; @@ -236,23 +236,6 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi return new TransportAddress[0]; } - @Override - public boolean nodeConnected(DiscoveryNode node) { - return true; - } - - @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - - } - - @Override - public void disconnectFromNode(DiscoveryNode node) { - - } - @Override public Lifecycle.State lifecycleState() { return null; @@ -283,11 +266,7 @@ public List getLocalAddresses() { } public Connection getConnection(DiscoveryNode node) { - try { - return openConnection(node, null); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + return openConnection(node, null); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 7f818de29d430..85a3c1831d28c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -20,11 +20,9 @@ package org.elasticsearch.test.transport; import com.carrotsearch.randomizedtesting.SysGlobals; - import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.component.Lifecycle; @@ -220,17 +218,7 @@ public void addFailToSendNoConnectRule(TransportAddress transportAddress) { addDelegate(transportAddress, new DelegateTransport(original) { @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - if (original.nodeConnected(node) == false) { - // connecting to an already connected node is a no-op - throw new ConnectTransportException(node, "DISCONNECT: simulated"); - } - } - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { throw new ConnectTransportException(node, "DISCONNECT: simulated"); } @@ -304,17 +292,7 @@ public void addUnresponsiveRule(TransportAddress transportAddress) { addDelegate(transportAddress, new DelegateTransport(original) { @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - if (original.nodeConnected(node) == false) { - // connecting to an already connected node is a no-op - throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); - } - } - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); } @@ -356,36 +334,7 @@ TimeValue getDelay() { } @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - if (original.nodeConnected(node)) { - // connecting to an already connected node is a no-op - return; - } - TimeValue delay = getDelay(); - if (delay.millis() <= 0) { - original.connectToNode(node, connectionProfile, connectionValidator); - return; - } - - // TODO: Replace with proper setting - TimeValue connectingTimeout = TcpTransport.TCP_CONNECT_TIMEOUT.getDefault(Settings.EMPTY); - try { - if (delay.millis() < connectingTimeout.millis()) { - Thread.sleep(delay.millis()); - original.connectToNode(node, connectionProfile, connectionValidator); - } else { - Thread.sleep(connectingTimeout.millis()); - throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); - } - } catch (InterruptedException e) { - throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); - } - } - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { TimeValue delay = getDelay(); if (delay.millis() <= 0) { return original.openConnection(node, profile); @@ -507,29 +456,7 @@ private Transport getTransport(DiscoveryNode node) { } @Override - public boolean nodeConnected(DiscoveryNode node) { - return getTransport(node).nodeConnected(node); - } - - @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - getTransport(node).connectToNode(node, connectionProfile, connectionValidator); - } - - @Override - public void disconnectFromNode(DiscoveryNode node) { - getTransport(node).disconnectFromNode(node); - } - - @Override - public Connection getConnection(DiscoveryNode node) { - return getTransport(node).getConnection(node); - } - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { return getTransport(node).openConnection(node, profile); } } @@ -577,41 +504,13 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi return transport.addressesFromString(address, perAddressLimit); } - @Override - public boolean nodeConnected(DiscoveryNode node) { - return transport.nodeConnected(node); - } - - @Override - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - transport.connectToNode(node, connectionProfile, connectionValidator); - } - - @Override - public void disconnectFromNode(DiscoveryNode node) { - transport.disconnectFromNode(node); - } - @Override public List getLocalAddresses() { return transport.getLocalAddresses(); } @Override - public Connection getConnection(DiscoveryNode node) { - return new FilteredConnection(transport.getConnection(node)) { - @Override - public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws IOException, TransportException { - DelegateTransport.this.sendRequest(connection, requestId, action, request, options); - } - }; - } - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { return new FilteredConnection(transport.openConnection(node, profile)) { @Override public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) From 744375e8235d8edf6556ba194f920f2c2c4d0f2b Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 6 Jul 2018 17:03:04 -0600 Subject: [PATCH 02/29] WIP --- .../transport/netty4/Netty4Transport.java | 8 +- .../netty4/Netty4ScheduledPingTests.java | 31 ++--- .../transport/ConnectionManager.java | 66 ++++++++- .../transport/RemoteClusterConnection.java | 5 + .../elasticsearch/transport/TcpTransport.java | 125 +++++------------- .../transport/TransportService.java | 4 +- 6 files changed, 124 insertions(+), 115 deletions(-) diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java index c8c6fceb54304..90138bc59906e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java @@ -267,8 +267,12 @@ protected Netty4TcpServerChannel bind(String name, InetSocketAddress address) { return esChannel; } - ScheduledPing getPing() { - return scheduledPing; + long successfulPingCount() { + return successfulPings.count(); + } + + long failedPingCount() { + return failedPings.count(); } @Override diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java index 01c5f5b617077..d95c82ac0c0cd 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java @@ -83,22 +83,19 @@ public void testScheduledPing() throws Exception { serviceB.connectToNode(nodeA); assertBusy(() -> { - assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(100L)); - assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(100L)); + assertThat(nettyA.successfulPingCount(), greaterThan(100L)); + assertThat(nettyB.successfulPingCount(), greaterThan(100L)); }); - assertThat(nettyA.getPing().getFailedPings(), equalTo(0L)); - assertThat(nettyB.getPing().getFailedPings(), equalTo(0L)); + assertThat(nettyA.failedPingCount(), equalTo(0L)); + assertThat(nettyB.failedPingCount(), equalTo(0L)); serviceA.registerRequestHandler("internal:sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC, - new TransportRequestHandler() { - @Override - public void messageReceived(TransportRequest.Empty request, TransportChannel channel, Task task) { - try { - channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.EMPTY); - } catch (IOException e) { - logger.error("Unexpected failure", e); - fail(e.getMessage()); - } + (request, channel, task) -> { + try { + channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.EMPTY); + } catch (IOException e) { + logger.error("Unexpected failure", e); + fail(e.getMessage()); } }); @@ -130,11 +127,11 @@ public void handleException(TransportException exp) { } assertBusy(() -> { - assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(200L)); - assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(200L)); + assertThat(nettyA.successfulPingCount(), greaterThan(200L)); + assertThat(nettyB.successfulPingCount(), greaterThan(200L)); }); - assertThat(nettyA.getPing().getFailedPings(), equalTo(0L)); - assertThat(nettyB.getPing().getFailedPings(), equalTo(0L)); + assertThat(nettyA.failedPingCount(), equalTo(0L)); + assertThat(nettyB.failedPingCount(), equalTo(0L)); Releasables.close(serviceA, serviceB); terminate(threadPool); diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index daa22d9c0c2ef..0df7d523277a1 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -22,15 +22,20 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.KeyedLock; import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; @@ -38,19 +43,30 @@ public class ConnectionManager { private final ConcurrentMap connectedNodes = newConcurrentMap(); private final KeyedLock connectionLock = new KeyedLock<>(); - private final AtomicBoolean isClosed = new AtomicBoolean(false); private final Logger logger; + private final Transport transport; + private final ThreadPool threadPool; + private final TimeValue pingSchedule; + private final Lifecycle lifecycle = new Lifecycle(); private final DelegatingNodeConnectionListener connectionListener = new DelegatingNodeConnectionListener(); - public ConnectionManager(Logger logger) { + public ConnectionManager(Logger logger, Transport transport, ThreadPool threadPool, TimeValue pingSchedule) { this.logger = logger; + this.transport = transport; + this.threadPool = threadPool; + this.pingSchedule = pingSchedule; + this.lifecycle.moveToStarted(); + + if (pingSchedule.millis() > 0 && transport instanceof TcpTransport) { + threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, new ScheduledPing()); + } } public void registerListener(TransportConnectionListener.NodeConnectionListener listener) { this.connectionListener.listeners.add(listener); } - public void connectToNode(Transport transport, DiscoveryNode node, ConnectionProfile connectionProfile, + public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, CheckedBiConsumer connectionValidator) throws ConnectTransportException { if (node == null) { @@ -124,11 +140,51 @@ public void disconnectFromNode(DiscoveryNode node) { } private void ensureOpen() { - if (isClosed.get()) { + if (lifecycle.started() == false) { throw new IllegalStateException("connection manager is closed"); } } + private class ScheduledPing extends AbstractLifecycleRunnable { + + private ScheduledPing() { + super(lifecycle, logger); + } + + @Override + protected void doRunInLifecycle() { + for (Map.Entry entry : connectedNodes.entrySet()) { + Transport.Connection connection = entry.getValue(); + if (connection instanceof TcpTransport.NodeChannels) { + TcpTransport.NodeChannels channels = (TcpTransport.NodeChannels) connection; + channels.sendPing(); + } + } + } + + @Override + protected void onAfterInLifecycle() { + try { + threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, this); + } catch (EsRejectedExecutionException ex) { + if (ex.isExecutorShutdown()) { + logger.debug("couldn't schedule new ping execution, executor is shutting down", ex); + } else { + throw ex; + } + } + } + + @Override + public void onFailure(Exception e) { + if (lifecycle.stoppedOrClosed()) { + logger.trace("failed to send ping transport message", e); + } else { + logger.warn("failed to send ping transport message", e); + } + } + } + private static final class DelegatingNodeConnectionListener implements TransportConnectionListener.NodeConnectionListener { private final List listeners = new CopyOnWriteArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 91d7b1b372e51..0add649dd7ef1 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -294,6 +294,11 @@ public void close() { assert false: "proxy connections must not be closed"; } + @Override + public boolean isClosed() { + return proxyConnection.isClosed(); + } + @Override public Version getVersion() { return proxyConnection.getVersion(); diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index 214a0d662739c..b2e038a0c0091 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -186,6 +186,8 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements // This is the number of bytes necessary to read the message size public static final int BYTES_NEEDED_FOR_MESSAGE_SIZE = TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE; public static final int PING_DATA_SIZE = -1; + protected final CounterMetric successfulPings = new CounterMetric(); + protected final CounterMetric failedPings = new CounterMetric(); private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.9); private static final BytesReference EMPTY_BYTES_REFERENCE = new BytesArray(new byte[0]); @@ -194,9 +196,6 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements private final String[] features; private final CircuitBreakerService circuitBreakerService; - // package visibility for tests - protected final ScheduledPing scheduledPing; - private final TimeValue pingSchedule; protected final ThreadPool threadPool; private final BigArrays bigArrays; protected final NetworkService networkService; @@ -226,6 +225,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements private final MeanMetric transmittedBytesMetric = new MeanMetric(); private volatile Map requestHandlers = Collections.emptyMap(); private final ResponseHandlers responseHandlers = new ResponseHandlers(); + private final BytesReference pingMessage; public TcpTransport(String transportName, Settings settings, ThreadPool threadPool, BigArrays bigArrays, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, @@ -235,8 +235,6 @@ public TcpTransport(String transportName, Settings settings, ThreadPool threadPo this.threadPool = threadPool; this.bigArrays = bigArrays; this.circuitBreakerService = circuitBreakerService; - this.scheduledPing = new ScheduledPing(); - this.pingSchedule = PING_SCHEDULE.get(settings); this.namedWriteableRegistry = namedWriteableRegistry; this.compress = Transport.TRANSPORT_TCP_COMPRESS.get(settings); this.networkService = networkService; @@ -254,6 +252,15 @@ public TcpTransport(String transportName, Settings settings, ThreadPool threadPo // use a sorted set to present the features in a consistent order this.features = new TreeSet<>(defaultFeatures.names()).toArray(new String[defaultFeatures.names().size()]); } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeByte((byte) 'E'); + out.writeByte((byte) 'S'); + out.writeInt(TcpTransport.PING_DATA_SIZE); + pingMessage = out.bytes(); + } catch (IOException e) { + throw new IllegalStateException(e.getMessage(), e); // won't happen + } } static ConnectionProfile buildDefaultConnectionProfile(Settings settings) { @@ -277,9 +284,6 @@ static ConnectionProfile buildDefaultConnectionProfile(Settings settings) { @Override protected void doStart() { - if (pingSchedule.millis() > 0) { - threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, scheduledPing); - } } @Override @@ -341,86 +345,6 @@ public String executor() { } } - public class ScheduledPing extends AbstractLifecycleRunnable { - - /** - * The magic number (must be lower than 0) for a ping message. - */ - private final BytesReference pingHeader; - final CounterMetric successfulPings = new CounterMetric(); - final CounterMetric failedPings = new CounterMetric(); - - public ScheduledPing() { - super(lifecycle, logger); - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeByte((byte) 'E'); - out.writeByte((byte) 'S'); - out.writeInt(PING_DATA_SIZE); - pingHeader = out.bytes(); - } catch (IOException e) { - throw new IllegalStateException(e.getMessage(), e); // won't happen - } - } - - @Override - protected void doRunInLifecycle() throws Exception { -// for (Map.Entry entry : connectedNodes.entrySet()) { -// DiscoveryNode node = entry.getKey(); -// NodeChannels channels = entry.getValue(); -// for (TcpChannel channel : channels.getChannels()) { -// internalSendMessage(channel, pingHeader, new SendMetricListener(pingHeader.length()) { -// @Override -// protected void innerInnerOnResponse(Void v) { -// successfulPings.inc(); -// } -// -// @Override -// protected void innerOnFailure(Exception e) { -// if (channel.isOpen()) { -// logger.debug(() -> new ParameterizedMessage("[{}] failed to send ping transport message", node), e); -// failedPings.inc(); -// } else { -// logger.trace(() -> -// new ParameterizedMessage("[{}] failed to send ping transport message (channel closed)", node), e); -// } -// -// } -// }); -// } -// } - } - - public long getSuccessfulPings() { - return successfulPings.count(); - } - - public long getFailedPings() { - return failedPings.count(); - } - - @Override - protected void onAfterInLifecycle() { - try { - threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, this); - } catch (EsRejectedExecutionException ex) { - if (ex.isExecutorShutdown()) { - logger.debug("couldn't schedule new ping execution, executor is shutting down", ex); - } else { - throw ex; - } - } - } - - @Override - public void onFailure(Exception e) { - if (lifecycle.stoppedOrClosed()) { - logger.trace("failed to send ping transport message", e); - } else { - logger.warn("failed to send ping transport message", e); - } - } - } - public final class NodeChannels implements Connection { private final Map typeMapping; private final List channels; @@ -458,10 +382,33 @@ public TcpChannel channel(TransportRequestOptions.Type type) { return connectionTypeHandle.getChannel(channels); } - public boolean allChannelsOpen() { + boolean allChannelsOpen() { return channels.stream().allMatch(TcpChannel::isOpen); } + void sendPing() { + for (TcpChannel channel : channels) { + internalSendMessage(channel, pingMessage, new SendMetricListener(pingMessage.length()) { + @Override + protected void innerInnerOnResponse(Void v) { + successfulPings.inc(); + } + + @Override + protected void innerOnFailure(Exception e) { + if (channel.isOpen()) { + logger.debug(() -> new ParameterizedMessage("[{}] failed to send ping transport message", node), e); + failedPings.inc(); + } else { + logger.trace(() -> + new ParameterizedMessage("[{}] failed to send ping transport message (channel closed)", node), e); + } + + } + }); + } + } + @Override public void close() { if (closed.compareAndSet(false, true)) { diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index a13ddf58314ab..cdc3d76caca11 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -153,7 +153,7 @@ public TransportService(Settings settings, Transport transport, ThreadPool threa this.transport = transport; this.threadPool = threadPool; this.localNodeFactory = localNodeFactory; - this.connectionManager = new ConnectionManager(logger); + this.connectionManager = new ConnectionManager(logger, transport, threadPool, TcpTransport.PING_SCHEDULE.get(settings)); this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings); setTracerLogInclude(TRACE_LOG_INCLUDE_SETTING.get(settings)); setTracerLogExclude(TRACE_LOG_EXCLUDE_SETTING.get(settings)); @@ -329,7 +329,7 @@ public void connectToNode(final DiscoveryNode node, ConnectionProfile connection if (isLocalNode(node)) { return; } - connectionManager.connectToNode(transport, node, connectionProfile, (newConnection, actualProfile) -> { + connectionManager.connectToNode(node, connectionProfile, (newConnection, actualProfile) -> { // We don't validate cluster names to allow for CCS connections. final DiscoveryNode remote = handshake(newConnection, actualProfile.getHandshakeTimeout().millis(), cn -> true).discoveryNode; if (validateConnections && node.equals(remote) == false) { From 002b371f93b9890cc88848ec4d76dc754191e366 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 6 Jul 2018 18:42:32 -0600 Subject: [PATCH 03/29] WIP --- .../discovery/zen/FaultDetection.java | 10 +++-- .../transport/ConnectionManager.java | 45 ++++++++++++++----- .../transport/RemoteClusterConnection.java | 18 +++++++- .../elasticsearch/transport/TcpTransport.java | 37 +++------------ .../elasticsearch/transport/Transport.java | 9 ++++ .../TransportConnectionListener.java | 26 ++++++----- .../transport/TransportService.java | 15 ++++++- .../TransportClientNodesServiceTests.java | 10 +++++ .../discovery/ZenFaultDetectionTests.java | 10 ++--- .../zen/PublishClusterStateActionTests.java | 10 ++--- .../transport/RemoteClusterClientTests.java | 7 ++- .../RemoteClusterConnectionTests.java | 9 +++- .../transport/RemoteClusterServiceTests.java | 6 ++- .../AbstractSimpleTransportTestCase.java | 28 ++++++------ 14 files changed, 156 insertions(+), 84 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java b/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java index 5d9b1687e4295..8733905affcb0 100644 --- a/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java +++ b/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java @@ -79,13 +79,13 @@ public FaultDetection(Settings settings, ThreadPool threadPool, TransportService this.connectionListener = new FDConnectionListener(); if (registerConnectionListener) { - transportService.addConnectionListener(connectionListener); + transportService.addNodeConnectionListener(connectionListener); } } @Override public void close() { - transportService.removeConnectionListener(connectionListener); + transportService.removeNodeConnectionListener(connectionListener); } /** @@ -93,7 +93,11 @@ public void close() { */ abstract void handleTransportDisconnect(DiscoveryNode node); - private class FDConnectionListener implements TransportConnectionListener { + private class FDConnectionListener implements TransportConnectionListener.NodeConnection { + @Override + public void onNodeConnected(DiscoveryNode node) { + } + @Override public void onNodeDisconnected(DiscoveryNode node) { AbstractRunnable runnable = new AbstractRunnable() { diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index 0df7d523277a1..255df6620113f 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -31,7 +31,9 @@ import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.threadpool.ThreadPool; +import java.io.Closeable; import java.io.IOException; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentMap; @@ -39,7 +41,7 @@ import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; -public class ConnectionManager { +public class ConnectionManager implements Closeable { private final ConcurrentMap connectedNodes = newConcurrentMap(); private final KeyedLock connectionLock = new KeyedLock<>(); @@ -57,12 +59,16 @@ public ConnectionManager(Logger logger, Transport transport, ThreadPool threadPo this.pingSchedule = pingSchedule; this.lifecycle.moveToStarted(); - if (pingSchedule.millis() > 0 && transport instanceof TcpTransport) { + if (pingSchedule.millis() > 0) { threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, new ScheduledPing()); } } - public void registerListener(TransportConnectionListener.NodeConnectionListener listener) { + public void addListener(TransportConnectionListener.NodeConnection listener) { + this.connectionListener.listeners.add(listener); + } + + public void removeListener(TransportConnectionListener.NodeConnection listener) { this.connectionListener.listeners.add(listener); } @@ -145,6 +151,26 @@ private void ensureOpen() { } } + @Override + public void close() { + lifecycle.moveToStopped(); + // TODO: Either add locking externally or in here. + // we are holding a write lock so nobody modifies the connectedNodes / openConnections map - it's safe to first close + // all instances and then clear them maps + Iterator> iterator = connectedNodes.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry next = iterator.next(); + try { + IOUtils.closeWhileHandlingException(next.getValue()); + connectionListener.onNodeDisconnected(next.getKey()); + } finally { + iterator.remove(); + } + } + + lifecycle.moveToClosed(); + } + private class ScheduledPing extends AbstractLifecycleRunnable { private ScheduledPing() { @@ -155,9 +181,8 @@ private ScheduledPing() { protected void doRunInLifecycle() { for (Map.Entry entry : connectedNodes.entrySet()) { Transport.Connection connection = entry.getValue(); - if (connection instanceof TcpTransport.NodeChannels) { - TcpTransport.NodeChannels channels = (TcpTransport.NodeChannels) connection; - channels.sendPing(); + if (connection.supportsPing()) { + connection.sendPing(); } } } @@ -185,20 +210,20 @@ public void onFailure(Exception e) { } } - private static final class DelegatingNodeConnectionListener implements TransportConnectionListener.NodeConnectionListener { + private static final class DelegatingNodeConnectionListener implements TransportConnectionListener.NodeConnection { - private final List listeners = new CopyOnWriteArrayList<>(); + private final List listeners = new CopyOnWriteArrayList<>(); @Override public void onNodeDisconnected(DiscoveryNode key) { - for (TransportConnectionListener.NodeConnectionListener listener : listeners) { + for (TransportConnectionListener.NodeConnection listener : listeners) { listener.onNodeDisconnected(key); } } @Override public void onNodeConnected(DiscoveryNode node) { - for (TransportConnectionListener.NodeConnectionListener listener : listeners) { + for (TransportConnectionListener.NodeConnection listener : listeners) { listener.onNodeConnected(node); } } diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 0add649dd7ef1..5ccf41eb54600 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -76,7 +76,7 @@ * {@link RemoteClusterService#REMOTE_CONNECTIONS_PER_CLUSTER} until either all eligible nodes are exhausted or the maximum number of * connections per cluster has been reached. */ -final class RemoteClusterConnection extends AbstractComponent implements TransportConnectionListener, Closeable { +final class RemoteClusterConnection extends AbstractComponent implements TransportConnectionListener.NodeConnection, Closeable { private final TransportService transportService; private final ConnectionProfile remoteProfile; @@ -121,7 +121,7 @@ final class RemoteClusterConnection extends AbstractComponent implements Transpo this.skipUnavailable = RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE .getConcreteSettingForNamespace(clusterAlias).get(settings); this.connectHandler = new ConnectHandler(); - transportService.addConnectionListener(this); + transportService.addNodeConnectionListener(this); } /** @@ -139,6 +139,10 @@ void updateSkipUnavailable(boolean skipUnavailable) { this.skipUnavailable = skipUnavailable; } + @Override + public void onNodeConnected(DiscoveryNode node) { + } + @Override public void onNodeDisconnected(DiscoveryNode node) { boolean remove = connectedNodes.remove(node); @@ -289,6 +293,16 @@ public void sendRequest(long requestId, String action, TransportRequest request, TransportActionProxy.wrapRequest(targetNode, request), options); } + @Override + public boolean supportsPing() { + return proxyConnection.supportsPing(); + } + + @Override + public void sendPing() { + proxyConnection.sendPing(); + } + @Override public void close() { assert false: "proxy connections must not be closed"; diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index b2e038a0c0091..d6a212f8b94a1 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -61,9 +61,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable; import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -386,7 +384,13 @@ boolean allChannelsOpen() { return channels.stream().allMatch(TcpChannel::isOpen); } - void sendPing() { + @Override + public boolean supportsPing() { + return true; + } + + @Override + public void sendPing() { for (TcpChannel channel : channels) { internalSendMessage(channel, pingMessage, new SendMetricListener(pingMessage.length()) { @Override @@ -820,19 +824,6 @@ protected final void doStop() { CloseableChannel.closeChannels(new ArrayList<>(acceptedChannels), true); acceptedChannels.clear(); - - // we are holding a write lock so nobody modifies the connectedNodes / openConnections map - it's safe to first close - // all instances and then clear them maps -// Iterator> iterator = connectedNodes.entrySet().iterator(); -// while (iterator.hasNext()) { -// Map.Entry next = iterator.next(); -// try { -// IOUtils.closeWhileHandlingException(next.getValue()); -// transportListener.onNodeDisconnected(next.getKey()); -// } finally { -// iterator.remove(); -// } -// } stopInternal(); } finally { closeLock.writeLock().unlock(); @@ -1789,13 +1780,6 @@ public void onRequestSent(DiscoveryNode node, long requestId, String action, Tra } } - @Override - public void onNodeDisconnected(DiscoveryNode key) { - for (TransportConnectionListener listener : listeners) { - listener.onNodeDisconnected(key); - } - } - @Override public void onConnectionOpened(Connection nodeChannels) { for (TransportConnectionListener listener : listeners) { @@ -1803,13 +1787,6 @@ public void onConnectionOpened(Connection nodeChannels) { } } - @Override - public void onNodeConnected(DiscoveryNode node) { - for (TransportConnectionListener listener : listeners) { - listener.onNodeConnected(node); - } - } - @Override public void onConnectionClosed(Connection nodeChannels) { for (TransportConnectionListener listener : listeners) { diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java index e5e4262106d34..6dc3cb04c9a65 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transport.java +++ b/server/src/main/java/org/elasticsearch/transport/Transport.java @@ -124,7 +124,16 @@ interface Connection extends Closeable { void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException; + default boolean supportsPing() { + return false; + } + + default void sendPing() { + throw new UnsupportedOperationException("Not support by this connection type"); + } + default boolean isClosed() { + // TODO: should probably not be default return false; } diff --git a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java index 5533fa07a814c..6d18ad99489c1 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java @@ -28,7 +28,7 @@ */ public interface TransportConnectionListener { - interface NodeConnectionListener { + interface NodeConnection { /** * Called once a node connection is opened and registered. */ @@ -40,6 +40,21 @@ interface NodeConnectionListener { void onNodeDisconnected(DiscoveryNode node); } + interface Connection { + + /** + * Called once a connection was opened + * @param connection the connection + */ + void onConnectionOpened(Transport.Connection connection); + + /** + * Called once a connection ws closed. + * @param connection the closed connection + */ + void onConnectionClosed(Transport.Connection connection); + } + /** * Called once a request is received * @param requestId the internal request ID @@ -95,13 +110,4 @@ default void onConnectionClosed(Transport.Connection connection) {} */ default void onResponseReceived(long requestId, Transport.ResponseContext context) {} - /** - * Called once a node connection is opened and registered. - */ - default void onNodeConnected(DiscoveryNode node) {} - - /** - * Called once a node connection is closed and unregistered. - */ - default void onNodeDisconnected(DiscoveryNode node) {} } diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index cdc3d76caca11..e7e82a5070a06 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -110,6 +110,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { Function.identity(), Property.Dynamic, Property.NodeScope); private final Logger tracerLog; + private final ConnectionProfile defaultConnectionProfile; volatile String[] tracerLogInclude; volatile String[] tracerLogExclude; @@ -164,6 +165,7 @@ public TransportService(Settings settings, Transport transport, ThreadPool threa this.connectToRemoteCluster = RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings); remoteClusterService = new RemoteClusterService(settings, this); responseHandlers = transport.getResponseHandlers(); + defaultConnectionProfile = TcpTransport.buildDefaultConnectionProfile(settings); if (clusterSettings != null) { clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_INCLUDE_SETTING, this::setTracerLogInclude); clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_EXCLUDE_SETTING, this::setTracerLogExclude); @@ -234,6 +236,7 @@ protected void doStart() { @Override protected void doStop() { try { + connectionManager.close(); transport.stop(); } finally { // in case the transport is not connected to our local node (thus cleaned on node disconnect) @@ -329,7 +332,9 @@ public void connectToNode(final DiscoveryNode node, ConnectionProfile connection if (isLocalNode(node)) { return; } - connectionManager.connectToNode(node, connectionProfile, (newConnection, actualProfile) -> { + + ConnectionProfile resolvedProfile = TcpTransport.resolveConnectionProfile(connectionProfile, defaultConnectionProfile); + connectionManager.connectToNode(node, resolvedProfile, (newConnection, actualProfile) -> { // We don't validate cluster names to allow for CCS connections. final DiscoveryNode remote = handshake(newConnection, actualProfile.getHandshakeTimeout().millis(), cn -> true).discoveryNode; if (validateConnections && node.equals(remote) == false) { @@ -467,6 +472,14 @@ public void disconnectFromNode(DiscoveryNode node) { connectionManager.disconnectFromNode(node); } + public void addNodeConnectionListener(NodeConnection listener) { + connectionManager.addListener(listener); + } + + public void removeNodeConnectionListener(NodeConnection listener) { + connectionManager.removeListener(listener); + } + public void addConnectionListener(TransportConnectionListener listener) { transport.addConnectionListener(listener); } diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index d4686e56adeb7..6027c8cf8920d 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -455,6 +455,16 @@ public void close() throws IOException { } } + @Override + public boolean supportsPing() { + return connection.supportsPing(); + } + + @Override + public void sendPing() { + connection.sendPing(); + } + @Override public boolean isClosed() { return closed.get(); diff --git a/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java index 03c0df43591ba..62eb332bedb04 100644 --- a/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java @@ -96,7 +96,7 @@ public void setUp() throws Exception { // wait till all nodes are properly connected and the event has been sent, so tests in this class // will not get this callback called on the connections done in this setup final CountDownLatch latch = new CountDownLatch(2); - TransportConnectionListener waitForConnection = new TransportConnectionListener() { + TransportConnectionListener.NodeConnection waitForConnection = new TransportConnectionListener.NodeConnection() { @Override public void onNodeConnected(DiscoveryNode node) { latch.countDown(); @@ -107,8 +107,8 @@ public void onNodeDisconnected(DiscoveryNode node) { fail("disconnect should not be called " + node); } }; - serviceA.addConnectionListener(waitForConnection); - serviceB.addConnectionListener(waitForConnection); + serviceA.addNodeConnectionListener(waitForConnection); + serviceB.addNodeConnectionListener(waitForConnection); serviceA.connectToNode(nodeB); serviceA.connectToNode(nodeA); @@ -116,8 +116,8 @@ public void onNodeDisconnected(DiscoveryNode node) { serviceB.connectToNode(nodeB); assertThat("failed to wait for all nodes to connect", latch.await(5, TimeUnit.SECONDS), equalTo(true)); - serviceA.removeConnectionListener(waitForConnection); - serviceB.removeConnectionListener(waitForConnection); + serviceA.removeNodeConnectionListener(waitForConnection); + serviceB.removeNodeConnectionListener(waitForConnection); } @Override diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java index ac1719269e7ae..06f32b7fdc821 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java @@ -190,7 +190,7 @@ public static MockNode createMockNode(String name, final Settings basSettings, @ MockNode node = new MockNode(discoveryNode, service, listener, logger); node.action = buildPublishClusterStateAction(settings, service, node); final CountDownLatch latch = new CountDownLatch(nodes.size() * 2); - TransportConnectionListener waitForConnection = new TransportConnectionListener() { + TransportConnectionListener.NodeConnection waitForConnection = new TransportConnectionListener.NodeConnection() { @Override public void onNodeConnected(DiscoveryNode node) { latch.countDown(); @@ -201,17 +201,17 @@ public void onNodeDisconnected(DiscoveryNode node) { fail("disconnect should not be called " + node); } }; - node.service.addConnectionListener(waitForConnection); + node.service.addNodeConnectionListener(waitForConnection); for (MockNode curNode : nodes.values()) { - curNode.service.addConnectionListener(waitForConnection); + curNode.service.addNodeConnectionListener(waitForConnection); curNode.connectTo(node.discoveryNode); node.connectTo(curNode.discoveryNode); } assertThat("failed to wait for all nodes to connect", latch.await(5, TimeUnit.SECONDS), equalTo(true)); for (MockNode curNode : nodes.values()) { - curNode.service.removeConnectionListener(waitForConnection); + curNode.service.removeNodeConnectionListener(waitForConnection); } - node.service.removeConnectionListener(waitForConnection); + node.service.removeNodeConnectionListener(waitForConnection); if (nodes.put(name, node) != null) { fail("Node with the name " + name + " already exist"); } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java index 8cfec0a07f910..90462e18a8801 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java @@ -81,7 +81,12 @@ public void testEnsureWeReconnect() throws Exception { try (MockTransportService service = MockTransportService.createNewService(localSettings, Version.CURRENT, threadPool, null)) { Semaphore semaphore = new Semaphore(1); service.start(); - service.addConnectionListener(new TransportConnectionListener() { + service.addNodeConnectionListener(new TransportConnectionListener.NodeConnection() { + @Override + public void onNodeConnected(DiscoveryNode node) { + + } + @Override public void onNodeDisconnected(DiscoveryNode node) { if (remoteNode.equals(node)) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 5e63356969fe8..af8ff4742a572 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -318,7 +318,7 @@ public void testNodeDisconnected() throws Exception { knownNodes.add(spareNode); CountDownLatch latchDisconnect = new CountDownLatch(1); CountDownLatch latchConnected = new CountDownLatch(1); - service.addConnectionListener(new TransportConnectionListener() { + service.addNodeConnectionListener(new TransportConnectionListener.NodeConnection() { @Override public void onNodeDisconnected(DiscoveryNode node) { if (node.equals(discoverableNode)) { @@ -645,7 +645,12 @@ public void testFetchShardsSkipUnavailable() throws Exception { } CountDownLatch disconnectedLatch = new CountDownLatch(1); - service.addConnectionListener(new TransportConnectionListener() { + service.addNodeConnectionListener(new TransportConnectionListener.NodeConnection() { + @Override + public void onNodeConnected(DiscoveryNode node) { + + } + @Override public void onNodeDisconnected(DiscoveryNode node) { if (node.equals(seedNode)) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java index 03d76b5a953c6..38590ec37b991 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java @@ -584,7 +584,11 @@ public void testCollectSearchShards() throws Exception { } CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters); - service.addConnectionListener(new TransportConnectionListener() { + service.addNodeConnectionListener(new TransportConnectionListener.NodeConnection() { + @Override + public void onNodeConnected(DiscoveryNode node) { + } + @Override public void onNodeDisconnected(DiscoveryNode node) { if (disconnectedNodes.remove(node)) { diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 50b7b8ce57597..aa50a17370ce2 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -128,7 +128,7 @@ public void setUp() throws Exception { // wait till all nodes are properly connected and the event has been sent, so tests in this class // will not get this callback called on the connections done in this setup final CountDownLatch latch = new CountDownLatch(2); - TransportConnectionListener waitForConnection = new TransportConnectionListener() { + TransportConnectionListener.NodeConnection waitForConnection = new TransportConnectionListener.NodeConnection() { @Override public void onNodeConnected(DiscoveryNode node) { latch.countDown(); @@ -139,8 +139,8 @@ public void onNodeDisconnected(DiscoveryNode node) { fail("disconnect should not be called " + node); } }; - serviceA.addConnectionListener(waitForConnection); - serviceB.addConnectionListener(waitForConnection); + serviceA.addNodeConnectionListener(waitForConnection); + serviceB.addNodeConnectionListener(waitForConnection); int numHandshakes = 1; serviceA.connectToNode(nodeB); serviceB.connectToNode(nodeA); @@ -148,8 +148,8 @@ public void onNodeDisconnected(DiscoveryNode node) { assertNumHandshakes(numHandshakes, serviceB.getOriginalTransport()); assertThat("failed to wait for all nodes to connect", latch.await(5, TimeUnit.SECONDS), equalTo(true)); - serviceA.removeConnectionListener(waitForConnection); - serviceB.removeConnectionListener(waitForConnection); + serviceA.removeNodeConnectionListener(waitForConnection); + serviceB.removeNodeConnectionListener(waitForConnection); } private MockTransportService buildService(final String name, final Version version, ClusterSettings clusterSettings, @@ -622,7 +622,7 @@ public void handleException(TransportException exp) { public void testDisconnectListener() throws Exception { final CountDownLatch latch = new CountDownLatch(1); - TransportConnectionListener disconnectListener = new TransportConnectionListener() { + TransportConnectionListener.NodeConnection disconnectListener = new TransportConnectionListener.NodeConnection() { @Override public void onNodeConnected(DiscoveryNode node) { fail("node connected should not be called, all connection have been done previously, node: " + node); @@ -633,7 +633,7 @@ public void onNodeDisconnected(DiscoveryNode node) { latch.countDown(); } }; - serviceA.addConnectionListener(disconnectListener); + serviceA.addNodeConnectionListener(disconnectListener); serviceB.close(); assertThat(latch.await(5, TimeUnit.SECONDS), equalTo(true)); } @@ -1704,7 +1704,7 @@ public void testSendRandomRequests() throws InterruptedException { serviceC.acceptIncomingRequests(); final CountDownLatch latch = new CountDownLatch(4); - TransportConnectionListener waitForConnection = new TransportConnectionListener() { + TransportConnectionListener.NodeConnection waitForConnection = new TransportConnectionListener.NodeConnection() { @Override public void onNodeConnected(DiscoveryNode node) { latch.countDown(); @@ -1715,9 +1715,9 @@ public void onNodeDisconnected(DiscoveryNode node) { fail("disconnect should not be called " + node); } }; - serviceA.addConnectionListener(waitForConnection); - serviceB.addConnectionListener(waitForConnection); - serviceC.addConnectionListener(waitForConnection); + serviceA.addNodeConnectionListener(waitForConnection); + serviceB.addNodeConnectionListener(waitForConnection); + serviceC.addNodeConnectionListener(waitForConnection); serviceC.connectToNode(nodeA); serviceC.connectToNode(nodeB); @@ -1725,9 +1725,9 @@ public void onNodeDisconnected(DiscoveryNode node) { serviceB.connectToNode(nodeC); latch.await(); - serviceA.removeConnectionListener(waitForConnection); - serviceB.removeConnectionListener(waitForConnection); - serviceC.removeConnectionListener(waitForConnection); + serviceA.removeNodeConnectionListener(waitForConnection); + serviceB.removeNodeConnectionListener(waitForConnection); + serviceC.removeNodeConnectionListener(waitForConnection); Map toNodeMap = new HashMap<>(); From 06a0093238966e27763b91c02d12c7b5dff27969 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 6 Jul 2018 19:07:09 -0600 Subject: [PATCH 04/29] At least fix checkstyle --- .../transport/netty4/Netty4ScheduledPingTests.java | 3 --- .../org/elasticsearch/transport/TcpTransport.java | 4 ++-- .../java/org/elasticsearch/transport/Transport.java | 1 - .../transport/TransportConnectionListener.java | 1 - .../client/transport/FailAndRetryMockTransport.java | 11 ----------- .../cluster/NodeConnectionsServiceTests.java | 3 --- .../discovery/zen/UnicastZenPingTests.java | 5 +---- .../test/transport/CapturingTransport.java | 3 --- 8 files changed, 3 insertions(+), 28 deletions(-) diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java index d95c82ac0c0cd..fae4082e81828 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java @@ -26,16 +26,13 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TcpTransport; -import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseHandler; diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index d6a212f8b94a1..9301b92fa9392 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -1781,14 +1781,14 @@ public void onRequestSent(DiscoveryNode node, long requestId, String action, Tra } @Override - public void onConnectionOpened(Connection nodeChannels) { + public void onConnectionOpened(Transport.Connection nodeChannels) { for (TransportConnectionListener listener : listeners) { listener.onConnectionOpened(nodeChannels); } } @Override - public void onConnectionClosed(Connection nodeChannels) { + public void onConnectionClosed(Transport.Connection nodeChannels) { for (TransportConnectionListener listener : listeners) { listener.onConnectionClosed(nodeChannels); } diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java index 6dc3cb04c9a65..b5798bde2741c 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transport.java +++ b/server/src/main/java/org/elasticsearch/transport/Transport.java @@ -21,7 +21,6 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.component.LifecycleComponent; diff --git a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java index 6d18ad99489c1..99b4121beb928 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java @@ -41,7 +41,6 @@ interface NodeConnection { } interface Connection { - /** * Called once a connection was opened * @param connection the connection diff --git a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java index 9ae722efcf457..4546f89a5f91c 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java +++ b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java @@ -19,33 +19,22 @@ package org.elasticsearch.client.transport; -import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse; -import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.RequestHandlerRegistry; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportConnectionListener; -import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; -import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportStats; -import java.io.IOException; import java.net.UnknownHostException; import java.util.Collections; import java.util.Map; diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index b55eb07547aeb..a96627f279e9d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; @@ -33,7 +32,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.RequestHandlerRegistry; import org.elasticsearch.transport.Transport; @@ -46,7 +44,6 @@ import org.junit.After; import org.junit.Before; -import java.io.IOException; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java index 95ad5e900485c..4ab738f5c7bc3 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java @@ -21,7 +21,6 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.Constants; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -29,7 +28,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode.Role; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.network.NetworkAddress; @@ -42,14 +40,13 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.ConnectTransportException; -import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.MockTcpTransport; import org.elasticsearch.transport.TcpTransport; import org.elasticsearch.transport.Transport; diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java index 0985b338f958f..c9f900577a8ba 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java @@ -21,7 +21,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; @@ -31,7 +30,6 @@ import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.RequestHandlerRegistry; @@ -45,7 +43,6 @@ import org.elasticsearch.transport.TransportStats; import java.io.IOException; -import java.io.UncheckedIOException; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; From f9e70807638e9cc022d5f9e0b4b450b9d60555c5 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Sun, 8 Jul 2018 18:36:02 -0600 Subject: [PATCH 05/29] Fix test --- .../java/org/elasticsearch/transport/ConnectionManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index 255df6620113f..1778c38386420 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -69,7 +69,7 @@ public void addListener(TransportConnectionListener.NodeConnection listener) { } public void removeListener(TransportConnectionListener.NodeConnection listener) { - this.connectionListener.listeners.add(listener); + this.connectionListener.listeners.remove(listener); } public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, From 67bc0b240e95099ad590a3bd697c861dd735f630 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Sun, 8 Jul 2018 18:42:20 -0600 Subject: [PATCH 06/29] Add comment --- .../main/java/org/elasticsearch/transport/ConnectionManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index 1778c38386420..bb8dec31b45b6 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -96,6 +96,7 @@ public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfil try { connectionListener.onNodeConnected(node); } finally { + // TODO: Need to add node disconnect listener if (connection.isClosed()) { // we got closed concurrently due to a disconnect or some other event on the channel. // the close callback will close the NodeChannel instance first and then try to remove From c3544adbf4e8d024cc5dc2c490608c0c446a7f6f Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jul 2018 12:30:26 -0600 Subject: [PATCH 07/29] Remove additional listeners --- .../discovery/zen/FaultDetection.java | 10 ++---- .../transport/ConnectionManager.java | 16 ++++----- .../transport/RemoteClusterConnection.java | 17 +++------ .../elasticsearch/transport/TcpTransport.java | 26 +++++++++----- .../elasticsearch/transport/Transport.java | 6 +--- .../TransportConnectionListener.java | 35 +++++-------------- .../transport/TransportService.java | 4 +-- .../TransportClientNodesServiceTests.java | 9 ++--- .../discovery/ZenFaultDetectionTests.java | 2 +- .../zen/PublishClusterStateActionTests.java | 2 +- .../transport/RemoteClusterClientTests.java | 7 +--- .../RemoteClusterConnectionTests.java | 9 ++--- .../transport/RemoteClusterServiceTests.java | 6 +--- .../AbstractSimpleTransportTestCase.java | 6 ++-- 14 files changed, 56 insertions(+), 99 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java b/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java index 8733905affcb0..5d9b1687e4295 100644 --- a/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java +++ b/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java @@ -79,13 +79,13 @@ public FaultDetection(Settings settings, ThreadPool threadPool, TransportService this.connectionListener = new FDConnectionListener(); if (registerConnectionListener) { - transportService.addNodeConnectionListener(connectionListener); + transportService.addConnectionListener(connectionListener); } } @Override public void close() { - transportService.removeNodeConnectionListener(connectionListener); + transportService.removeConnectionListener(connectionListener); } /** @@ -93,11 +93,7 @@ public void close() { */ abstract void handleTransportDisconnect(DiscoveryNode node); - private class FDConnectionListener implements TransportConnectionListener.NodeConnection { - @Override - public void onNodeConnected(DiscoveryNode node) { - } - + private class FDConnectionListener implements TransportConnectionListener { @Override public void onNodeDisconnected(DiscoveryNode node) { AbstractRunnable runnable = new AbstractRunnable() { diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index bb8dec31b45b6..a3a2ec4269cc7 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -64,11 +64,11 @@ public ConnectionManager(Logger logger, Transport transport, ThreadPool threadPo } } - public void addListener(TransportConnectionListener.NodeConnection listener) { + public void addListener(TransportConnectionListener listener) { this.connectionListener.listeners.add(listener); } - public void removeListener(TransportConnectionListener.NodeConnection listener) { + public void removeListener(TransportConnectionListener listener) { this.connectionListener.listeners.remove(listener); } @@ -182,8 +182,8 @@ private ScheduledPing() { protected void doRunInLifecycle() { for (Map.Entry entry : connectedNodes.entrySet()) { Transport.Connection connection = entry.getValue(); - if (connection.supportsPing()) { - connection.sendPing(); + if (connection.sendPing() == false) { + logger.warn("attempted to send ping to connection without support for pings [{}]", connection); } } } @@ -211,20 +211,20 @@ public void onFailure(Exception e) { } } - private static final class DelegatingNodeConnectionListener implements TransportConnectionListener.NodeConnection { + private static final class DelegatingNodeConnectionListener implements TransportConnectionListener { - private final List listeners = new CopyOnWriteArrayList<>(); + private final List listeners = new CopyOnWriteArrayList<>(); @Override public void onNodeDisconnected(DiscoveryNode key) { - for (TransportConnectionListener.NodeConnection listener : listeners) { + for (TransportConnectionListener listener : listeners) { listener.onNodeDisconnected(key); } } @Override public void onNodeConnected(DiscoveryNode node) { - for (TransportConnectionListener.NodeConnection listener : listeners) { + for (TransportConnectionListener listener : listeners) { listener.onNodeConnected(node); } } diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 5ccf41eb54600..335f61588c067 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -76,7 +76,7 @@ * {@link RemoteClusterService#REMOTE_CONNECTIONS_PER_CLUSTER} until either all eligible nodes are exhausted or the maximum number of * connections per cluster has been reached. */ -final class RemoteClusterConnection extends AbstractComponent implements TransportConnectionListener.NodeConnection, Closeable { +final class RemoteClusterConnection extends AbstractComponent implements TransportConnectionListener, Closeable { private final TransportService transportService; private final ConnectionProfile remoteProfile; @@ -121,7 +121,7 @@ final class RemoteClusterConnection extends AbstractComponent implements Transpo this.skipUnavailable = RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE .getConcreteSettingForNamespace(clusterAlias).get(settings); this.connectHandler = new ConnectHandler(); - transportService.addNodeConnectionListener(this); + transportService.addConnectionListener(this); } /** @@ -139,10 +139,6 @@ void updateSkipUnavailable(boolean skipUnavailable) { this.skipUnavailable = skipUnavailable; } - @Override - public void onNodeConnected(DiscoveryNode node) { - } - @Override public void onNodeDisconnected(DiscoveryNode node) { boolean remove = connectedNodes.remove(node); @@ -294,13 +290,8 @@ public void sendRequest(long requestId, String action, TransportRequest request, } @Override - public boolean supportsPing() { - return proxyConnection.supportsPing(); - } - - @Override - public void sendPing() { - proxyConnection.sendPing(); + public boolean sendPing() { + return proxyConnection.sendPing(); } @Override diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index 9301b92fa9392..2a2324e017410 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -385,12 +385,7 @@ boolean allChannelsOpen() { } @Override - public boolean supportsPing() { - return true; - } - - @Override - public void sendPing() { + public boolean sendPing() { for (TcpChannel channel : channels) { internalSendMessage(channel, pingMessage, new SendMetricListener(pingMessage.length()) { @Override @@ -411,6 +406,7 @@ protected void innerOnFailure(Exception e) { } }); } + return true; } @Override @@ -1781,14 +1777,28 @@ public void onRequestSent(DiscoveryNode node, long requestId, String action, Tra } @Override - public void onConnectionOpened(Transport.Connection nodeChannels) { + public void onNodeDisconnected(DiscoveryNode key) { + for (TransportConnectionListener listener : listeners) { + listener.onNodeDisconnected(key); + } + } + + @Override + public void onConnectionOpened(Connection nodeChannels) { for (TransportConnectionListener listener : listeners) { listener.onConnectionOpened(nodeChannels); } } @Override - public void onConnectionClosed(Transport.Connection nodeChannels) { + public void onNodeConnected(DiscoveryNode node) { + for (TransportConnectionListener listener : listeners) { + listener.onNodeConnected(node); + } + } + + @Override + public void onConnectionClosed(Connection nodeChannels) { for (TransportConnectionListener listener : listeners) { listener.onConnectionClosed(nodeChannels); } diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java index b5798bde2741c..1584d1f183b9e 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transport.java +++ b/server/src/main/java/org/elasticsearch/transport/Transport.java @@ -123,14 +123,10 @@ interface Connection extends Closeable { void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException; - default boolean supportsPing() { + default boolean sendPing() { return false; } - default void sendPing() { - throw new UnsupportedOperationException("Not support by this connection type"); - } - default boolean isClosed() { // TODO: should probably not be default return false; diff --git a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java index 99b4121beb928..0ee2ed5828d44 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java @@ -28,32 +28,6 @@ */ public interface TransportConnectionListener { - interface NodeConnection { - /** - * Called once a node connection is opened and registered. - */ - void onNodeConnected(DiscoveryNode node); - - /** - * Called once a node connection is closed and unregistered. - */ - void onNodeDisconnected(DiscoveryNode node); - } - - interface Connection { - /** - * Called once a connection was opened - * @param connection the connection - */ - void onConnectionOpened(Transport.Connection connection); - - /** - * Called once a connection ws closed. - * @param connection the closed connection - */ - void onConnectionClosed(Transport.Connection connection); - } - /** * Called once a request is received * @param requestId the internal request ID @@ -109,4 +83,13 @@ default void onConnectionClosed(Transport.Connection connection) {} */ default void onResponseReceived(long requestId, Transport.ResponseContext context) {} + /** + * Called once a node connection is opened and registered. + */ + default void onNodeConnected(DiscoveryNode node) {} + + /** + * Called once a node connection is closed and unregistered. + */ + default void onNodeDisconnected(DiscoveryNode node) {} } diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index e7e82a5070a06..68dd29210a84c 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -472,11 +472,11 @@ public void disconnectFromNode(DiscoveryNode node) { connectionManager.disconnectFromNode(node); } - public void addNodeConnectionListener(NodeConnection listener) { + public void addNodeConnectionListener(TransportConnectionListener listener) { connectionManager.addListener(listener); } - public void removeNodeConnectionListener(NodeConnection listener) { + public void removeNodeConnectionListener(TransportConnectionListener listener) { connectionManager.removeListener(listener); } diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 6027c8cf8920d..458ad9127f08f 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -456,13 +456,8 @@ public void close() throws IOException { } @Override - public boolean supportsPing() { - return connection.supportsPing(); - } - - @Override - public void sendPing() { - connection.sendPing(); + public boolean sendPing() { + return connection.sendPing(); } @Override diff --git a/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java index 62eb332bedb04..b573ecd6ef2d5 100644 --- a/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java @@ -96,7 +96,7 @@ public void setUp() throws Exception { // wait till all nodes are properly connected and the event has been sent, so tests in this class // will not get this callback called on the connections done in this setup final CountDownLatch latch = new CountDownLatch(2); - TransportConnectionListener.NodeConnection waitForConnection = new TransportConnectionListener.NodeConnection() { + TransportConnectionListener waitForConnection = new TransportConnectionListener() { @Override public void onNodeConnected(DiscoveryNode node) { latch.countDown(); diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java index 06f32b7fdc821..c9a634c7f9ce7 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java @@ -190,7 +190,7 @@ public static MockNode createMockNode(String name, final Settings basSettings, @ MockNode node = new MockNode(discoveryNode, service, listener, logger); node.action = buildPublishClusterStateAction(settings, service, node); final CountDownLatch latch = new CountDownLatch(nodes.size() * 2); - TransportConnectionListener.NodeConnection waitForConnection = new TransportConnectionListener.NodeConnection() { + TransportConnectionListener waitForConnection = new TransportConnectionListener() { @Override public void onNodeConnected(DiscoveryNode node) { latch.countDown(); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java index 90462e18a8801..9c81896624d63 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java @@ -81,12 +81,7 @@ public void testEnsureWeReconnect() throws Exception { try (MockTransportService service = MockTransportService.createNewService(localSettings, Version.CURRENT, threadPool, null)) { Semaphore semaphore = new Semaphore(1); service.start(); - service.addNodeConnectionListener(new TransportConnectionListener.NodeConnection() { - @Override - public void onNodeConnected(DiscoveryNode node) { - - } - + service.addNodeConnectionListener(new TransportConnectionListener() { @Override public void onNodeDisconnected(DiscoveryNode node) { if (remoteNode.equals(node)) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index af8ff4742a572..559ffa7655d09 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -318,7 +318,7 @@ public void testNodeDisconnected() throws Exception { knownNodes.add(spareNode); CountDownLatch latchDisconnect = new CountDownLatch(1); CountDownLatch latchConnected = new CountDownLatch(1); - service.addNodeConnectionListener(new TransportConnectionListener.NodeConnection() { + service.addNodeConnectionListener(new TransportConnectionListener() { @Override public void onNodeDisconnected(DiscoveryNode node) { if (node.equals(discoverableNode)) { @@ -645,12 +645,7 @@ public void testFetchShardsSkipUnavailable() throws Exception { } CountDownLatch disconnectedLatch = new CountDownLatch(1); - service.addNodeConnectionListener(new TransportConnectionListener.NodeConnection() { - @Override - public void onNodeConnected(DiscoveryNode node) { - - } - + service.addNodeConnectionListener(new TransportConnectionListener() { @Override public void onNodeDisconnected(DiscoveryNode node) { if (node.equals(seedNode)) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java index 38590ec37b991..aa4ed509952cc 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java @@ -584,11 +584,7 @@ public void testCollectSearchShards() throws Exception { } CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters); - service.addNodeConnectionListener(new TransportConnectionListener.NodeConnection() { - @Override - public void onNodeConnected(DiscoveryNode node) { - } - + service.addNodeConnectionListener(new TransportConnectionListener() { @Override public void onNodeDisconnected(DiscoveryNode node) { if (disconnectedNodes.remove(node)) { diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index aa50a17370ce2..b821d36364d56 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -128,7 +128,7 @@ public void setUp() throws Exception { // wait till all nodes are properly connected and the event has been sent, so tests in this class // will not get this callback called on the connections done in this setup final CountDownLatch latch = new CountDownLatch(2); - TransportConnectionListener.NodeConnection waitForConnection = new TransportConnectionListener.NodeConnection() { + TransportConnectionListener waitForConnection = new TransportConnectionListener() { @Override public void onNodeConnected(DiscoveryNode node) { latch.countDown(); @@ -622,7 +622,7 @@ public void handleException(TransportException exp) { public void testDisconnectListener() throws Exception { final CountDownLatch latch = new CountDownLatch(1); - TransportConnectionListener.NodeConnection disconnectListener = new TransportConnectionListener.NodeConnection() { + TransportConnectionListener disconnectListener = new TransportConnectionListener() { @Override public void onNodeConnected(DiscoveryNode node) { fail("node connected should not be called, all connection have been done previously, node: " + node); @@ -1704,7 +1704,7 @@ public void testSendRandomRequests() throws InterruptedException { serviceC.acceptIncomingRequests(); final CountDownLatch latch = new CountDownLatch(4); - TransportConnectionListener.NodeConnection waitForConnection = new TransportConnectionListener.NodeConnection() { + TransportConnectionListener waitForConnection = new TransportConnectionListener() { @Override public void onNodeConnected(DiscoveryNode node) { latch.countDown(); From fc886bfb8128fa38f72726f815bb972c6e3a299c Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jul 2018 16:51:17 -0600 Subject: [PATCH 08/29] Work on fixing tests --- .../cluster/NodeConnectionsService.java | 2 +- .../transport/ConnectionManager.java | 25 ++-- .../transport/RemoteClusterConnection.java | 5 + .../elasticsearch/transport/TcpTransport.java | 16 ++- .../elasticsearch/transport/Transport.java | 6 + .../transport/TransportService.java | 8 +- .../transport/FailAndRetryMockTransport.java | 131 +++++++++--------- .../TransportClientNodesServiceTests.java | 5 + .../cluster/NodeConnectionsServiceTests.java | 104 ++++++-------- .../RemoteClusterConnectionTests.java | 100 ++++++------- 10 files changed, 206 insertions(+), 196 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java index 998cd5ba0a870..2a9d960f8ccea 100644 --- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java @@ -101,7 +101,7 @@ public void onFailure(Exception e) { } @Override - protected void doRun() throws Exception { + protected void doRun() { try (Releasable ignored = nodeLocks.acquire(node)) { validateAndConnectIfNeeded(node); } diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index a3a2ec4269cc7..c40bd7cb13ad9 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -20,6 +20,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.component.Lifecycle; @@ -96,21 +97,15 @@ public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfil try { connectionListener.onNodeConnected(node); } finally { - // TODO: Need to add node disconnect listener - if (connection.isClosed()) { - // we got closed concurrently due to a disconnect or some other event on the channel. - // the close callback will close the NodeChannel instance first and then try to remove - // the connection from the connected nodes. It will NOT acquire the connectionLock for - // the node to prevent any blocking calls on network threads. Yet, we still establish a happens - // before relationship to the connectedNodes.put since we check if we can remove the - // (DiscoveryNode, NodeChannels) tuple from the map after we closed. Here we check if it's closed an if so we - // try to remove it first either way one of the two wins even if the callback has run before we even added the - // tuple to the map since in that case we remove it here again - if (connectedNodes.remove(node, connection)) { + final Transport.Connection finalConnection = connection; + connection.addCloseListener(ActionListener.wrap(() -> { + if (connectedNodes.remove(node, finalConnection)) { connectionListener.onNodeDisconnected(node); } - throw new NodeNotConnectedException(node, "connection concurrently closed"); - } + })); + } + if (connection.isClosed()) { + throw new NodeNotConnectedException(node, "connection concurrently closed"); } success = true; } catch (ConnectTransportException e) { @@ -172,6 +167,10 @@ public void close() { lifecycle.moveToClosed(); } + public int connectedNodeCount() { + return connectedNodes.size(); + } + private class ScheduledPing extends AbstractLifecycleRunnable { private ScheduledPing() { diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 335f61588c067..355a9c655c998 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -299,6 +299,11 @@ public void close() { assert false: "proxy connections must not be closed"; } + @Override + public void addCloseListener(ActionListener listener) { + proxyConnection.addCloseListener(listener); + } + @Override public boolean isClosed() { return proxyConnection.isClosed(); diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index 2a2324e017410..5868c0d968eaa 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.compress.NotCompressedException; +import org.elasticsearch.common.concurrent.CompletableContext; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -257,7 +258,7 @@ public TcpTransport(String transportName, Settings settings, ThreadPool threadPo out.writeInt(TcpTransport.PING_DATA_SIZE); pingMessage = out.bytes(); } catch (IOException e) { - throw new IllegalStateException(e.getMessage(), e); // won't happen + throw new AssertionError(e.getMessage(), e); // won't happen } } @@ -347,8 +348,8 @@ public final class NodeChannels implements Connection { private final Map typeMapping; private final List channels; private final DiscoveryNode node; - private final AtomicBoolean closed = new AtomicBoolean(false); private final Version version; + private final CompletableContext closeContext = new CompletableContext<>(); NodeChannels(DiscoveryNode node, List channels, ConnectionProfile connectionProfile, Version handshakeVersion) { this.node = node; @@ -409,9 +410,14 @@ protected void innerOnFailure(Exception e) { return true; } + @Override + public void addCloseListener(ActionListener listener) { + closeContext.addListener(ActionListener.toBiConsumer(listener)); + } + @Override public void close() { - if (closed.compareAndSet(false, true)) { + if (closeContext.complete(null)) { try { if (lifecycle.stopped()) { /* We set SO_LINGER timeout to 0 to ensure that when we shutdown the node we don't @@ -447,7 +453,7 @@ public DiscoveryNode getNode() { @Override public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { - if (closed.get()) { + if (closeContext.isDone()) { throw new NodeNotConnectedException(node, "connection already closed"); } TcpChannel channel = channel(options.type()); @@ -456,7 +462,7 @@ public void sendRequest(long requestId, String action, TransportRequest request, @Override public boolean isClosed() { - return closed.get(); + return closeContext.isDone(); } } diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java index 1584d1f183b9e..ab278546da4b7 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transport.java +++ b/server/src/main/java/org/elasticsearch/transport/Transport.java @@ -20,6 +20,7 @@ package org.elasticsearch.transport; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.NoopCircuitBreaker; @@ -127,6 +128,11 @@ default boolean sendPing() { return false; } + default void addCloseListener(ActionListener listener) { + throw new UnsupportedOperationException("Not supported"); + } + + default boolean isClosed() { // TODO: should probably not be default return false; diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 68dd29210a84c..f1bb283168deb 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -21,6 +21,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.action.termvectors.TermVectorsFilter; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.core.internal.io.IOUtils; @@ -385,12 +386,11 @@ public DiscoveryNode handshake( * @param handshakeTimeout handshake timeout * @param clusterNamePredicate cluster name validation predicate * @return the handshake response - * @throws ConnectTransportException if the connection failed * @throws IllegalStateException if the handshake failed */ public HandshakeResponse handshake( final Transport.Connection connection, - final long handshakeTimeout, Predicate clusterNamePredicate) throws ConnectTransportException { + final long handshakeTimeout, Predicate clusterNamePredicate) { final HandshakeResponse response; final DiscoveryNode node = connection.getNode(); try { @@ -417,6 +417,10 @@ public HandshakeResponse newInstance() { return response; } + public ConnectionManager getConnectionManager() { + return connectionManager; + } + static class HandshakeRequest extends TransportRequest { public static final HandshakeRequest INSTANCE = new HandshakeRequest(); diff --git a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java index 4546f89a5f91c..5dd2f8db506ee 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java +++ b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java @@ -19,20 +19,29 @@ package org.elasticsearch.client.transport; +import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse; +import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.RequestHandlerRegistry; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportConnectionListener; +import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportStats; import java.net.UnknownHostException; @@ -48,7 +57,7 @@ abstract class FailAndRetryMockTransport imp private final Random random; private final ClusterName clusterName; private volatile Map requestHandlers = Collections.emptyMap(); - final Object requestHandlerMutex = new Object(); + private final Object requestHandlerMutex = new Object(); private final ResponseHandlers responseHandlers = new ResponseHandlers(); private TransportConnectionListener listener; @@ -66,70 +75,66 @@ abstract class FailAndRetryMockTransport imp protected abstract ClusterState getMockClusterState(DiscoveryNode node); -// @Override -// public Connection getConnection(DiscoveryNode node) { -// return new Connection() { -// @Override -// public DiscoveryNode getNode() { -// return node; -// } -// -// @Override -// public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) -// throws IOException, TransportException { -// -// //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info -// if (connectMode) { -// if (TransportLivenessAction.NAME.equals(action)) { -// TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); -// transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.CLUSTER_NAME_SETTING. -// getDefault(Settings.EMPTY), -// node)); -// } else if (ClusterStateAction.NAME.equals(action)) { -// TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); -// ClusterState clusterState = getMockClusterState(node); -// transportResponseHandler.handleResponse(new ClusterStateResponse(clusterName, clusterState, 0L)); -// } else { -// throw new UnsupportedOperationException("Mock transport does not understand action " + action); -// } -// return; -// } -// -// //once nodes are connected we'll just return errors for each sendRequest call -// triedNodes.add(node); -// -// if (random.nextInt(100) > 10) { -// connectTransportExceptions.incrementAndGet(); -// throw new ConnectTransportException(node, "node not available"); -// } else { -// if (random.nextBoolean()) { -// failures.incrementAndGet(); -// //throw whatever exception that is not a subclass of ConnectTransportException -// throw new IllegalStateException(); -// } else { -// TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); -// if (random.nextBoolean()) { -// successes.incrementAndGet(); -// transportResponseHandler.handleResponse(newResponse()); -// } else { -// failures.incrementAndGet(); -// transportResponseHandler.handleException(new TransportException("transport exception")); -// } -// } -// } -// } -// -// @Override -// public void close() { -// -// } -// }; -// } - @Override public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { -// return getConnection(node); - return null; + return new Connection() { + @Override + public DiscoveryNode getNode() { + return node; + } + + @Override + public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) + throws TransportException { + //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info + if (connectMode) { + if (TransportLivenessAction.NAME.equals(action)) { + TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); + ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY); + transportResponseHandler.handleResponse(new LivenessResponse(clusterName, node)); + } else if (ClusterStateAction.NAME.equals(action)) { + TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); + ClusterState clusterState = getMockClusterState(node); + transportResponseHandler.handleResponse(new ClusterStateResponse(clusterName, clusterState, 0L)); + } else { + throw new UnsupportedOperationException("Mock transport does not understand action " + action); + } + return; + } + + //once nodes are connected we'll just return errors for each sendRequest call + triedNodes.add(node); + + if (random.nextInt(100) > 10) { + connectTransportExceptions.incrementAndGet(); + throw new ConnectTransportException(node, "node not available"); + } else { + if (random.nextBoolean()) { + failures.incrementAndGet(); + //throw whatever exception that is not a subclass of ConnectTransportException + throw new IllegalStateException(); + } else { + TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); + if (random.nextBoolean()) { + successes.incrementAndGet(); + transportResponseHandler.handleResponse(newResponse()); + } else { + failures.incrementAndGet(); + transportResponseHandler.handleException(new TransportException("transport exception")); + } + } + } + } + + @Override + public void close() { + } + + @Override + public boolean isClosed() { + return false; + } + }; } protected abstract Response newResponse(); diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 458ad9127f08f..f06f3a8a66d1f 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -460,6 +460,11 @@ public boolean sendPing() { return connection.sendPing(); } + @Override + public void addCloseListener(ActionListener listener) { + connection.addCloseListener(listener); + } + @Override public boolean isClosed() { return closed.get(); diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index a96627f279e9d..d64c903c3442b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -25,18 +25,20 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; +import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.RequestHandlerRegistry; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportConnectionListener; import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportInterceptor; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; @@ -44,7 +46,6 @@ import org.junit.After; import org.junit.Before; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -53,6 +54,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.Predicate; import static org.hamcrest.Matchers.equalTo; @@ -118,7 +121,7 @@ public void testReconnect() { for (int i = 0; i < 3; i++) { // simulate disconnects for (DiscoveryNode node : randomSubsetOf(nodes)) { -// transport.disconnectFromNode(node); + transportService.disconnectFromNode(node); } service.new ConnectionChecker().run(); } @@ -131,18 +134,12 @@ public void testReconnect() { private void assertConnectedExactlyToNodes(ClusterState state) { assertConnected(state.nodes()); - assertThat(transport.connectedNodes.size(), equalTo(state.nodes().getSize())); + assertThat(transportService.getConnectionManager().connectedNodeCount(), equalTo(state.nodes().getSize())); } private void assertConnected(Iterable nodes) { for (DiscoveryNode node : nodes) { - assertTrue("not connected to " + node, transport.connectedNodes.contains(node)); - } - } - - private void assertNotConnected(Iterable nodes) { - for (DiscoveryNode node : nodes) { - assertFalse("still connected to " + node, transport.connectedNodes.contains(node)); + assertTrue("not connected to " + node, transportService.nodeConnected(node)); } } @@ -152,7 +149,7 @@ public void setUp() throws Exception { super.setUp(); this.threadPool = new TestThreadPool(getClass().getName()); this.transport = new MockTransport(); - transportService = new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, + transportService = new NoHandshakeTransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), UUIDs.randomBase64UUID()), null, Collections.emptySet()); transportService.start(); @@ -168,10 +165,27 @@ public void tearDown() throws Exception { super.tearDown(); } - final class MockTransport implements Transport { - Set connectedNodes = ConcurrentCollections.newConcurrentSet(); - volatile boolean randomConnectionExceptions = false; + private final class NoHandshakeTransportService extends TransportService { + + public NoHandshakeTransportService(Settings settings, + Transport transport, + ThreadPool threadPool, + TransportInterceptor transportInterceptor, + Function localNodeFactory, + ClusterSettings clusterSettings, + Set taskHeaders) { + super(settings, transport, threadPool, transportInterceptor, localNodeFactory, clusterSettings, taskHeaders); + } + + @Override + public HandshakeResponse handshake(Transport.Connection connection, long timeout, Predicate clusterNamePredicate) { + return new HandshakeResponse(connection.getNode(), new ClusterName(""), Version.CURRENT); + } + } + + private final class MockTransport implements Transport { private ResponseHandlers responseHandlers = new ResponseHandlers(); + private volatile boolean randomConnectionExceptions = false; private TransportConnectionListener listener = new TransportConnectionListener() { }; @@ -205,57 +219,18 @@ public Map profileBoundAddresses() { } @Override - public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException { + public TransportAddress[] addressesFromString(String address, int perAddressLimit) { return new TransportAddress[0]; } -// @Override -// public boolean nodeConnected(DiscoveryNode node) { -// return connectedNodes.contains(node); -// } - -// @Override -// public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, -// CheckedBiConsumer connectionValidator) -// throws ConnectTransportException { -// if (connectionProfile == null) { -// if (connectedNodes.contains(node) == false && randomConnectionExceptions && randomBoolean()) { -// throw new ConnectTransportException(node, "simulated"); -// } -// connectedNodes.add(node); -// listener.onNodeConnected(node); -// } -// } -// -// @Override -// public void disconnectFromNode(DiscoveryNode node) { -// connectedNodes.remove(node); -// listener.onNodeDisconnected(node); -// } - -// @Override -// public Connection getConnection(DiscoveryNode node) { -// return new Connection() { -// @Override -// public DiscoveryNode getNode() { -// return node; -// } -// -// @Override -// public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) -// throws TransportException { -// -// } -// -// @Override -// public void close() { -// -// } -// }; -// } - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { + public Connection openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) { + if (connectionProfile == null) { + if (randomConnectionExceptions && randomBoolean()) { + throw new ConnectTransportException(node, "simulated"); + } + listener.onNodeConnected(node); + } Connection connection = new Connection() { @Override public DiscoveryNode getNode() { @@ -272,6 +247,11 @@ public void sendRequest(long requestId, String action, TransportRequest request, public void close() { } + + @Override + public boolean isClosed() { + return false; + } }; listener.onConnectionOpened(connection); return connection; diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 559ffa7655d09..22ae194ede056 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -294,55 +294,55 @@ public void testDiscoverSingleNodeWithIncompatibleSeed() throws Exception { } } - public void testNodeDisconnected() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT); - MockTransportService spareTransport = startTransport("spare_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - DiscoveryNode spareNode = spareTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) { - updateSeedNodes(connection, Arrays.asList(seedNode)); - assertTrue(service.nodeConnected(seedNode)); - assertTrue(service.nodeConnected(discoverableNode)); - assertFalse(service.nodeConnected(spareNode)); - knownNodes.add(spareNode); - CountDownLatch latchDisconnect = new CountDownLatch(1); - CountDownLatch latchConnected = new CountDownLatch(1); - service.addNodeConnectionListener(new TransportConnectionListener() { - @Override - public void onNodeDisconnected(DiscoveryNode node) { - if (node.equals(discoverableNode)) { - latchDisconnect.countDown(); - } - } - - @Override - public void onNodeConnected(DiscoveryNode node) { - if (node.equals(spareNode)) { - latchConnected.countDown(); - } - } - }); - - discoverableTransport.close(); - // now make sure we try to connect again to other nodes once we got disconnected - assertTrue(latchDisconnect.await(10, TimeUnit.SECONDS)); - assertTrue(latchConnected.await(10, TimeUnit.SECONDS)); - assertTrue(service.nodeConnected(spareNode)); - } - } - } - } +// public void testNodeDisconnected() throws Exception { +// List knownNodes = new CopyOnWriteArrayList<>(); +// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); +// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT); +// MockTransportService spareTransport = startTransport("spare_node", knownNodes, Version.CURRENT)) { +// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); +// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); +// DiscoveryNode spareNode = spareTransport.getLocalDiscoNode(); +// knownNodes.add(seedTransport.getLocalDiscoNode()); +// knownNodes.add(discoverableTransport.getLocalDiscoNode()); +// Collections.shuffle(knownNodes, random()); +// +// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { +// service.start(); +// service.acceptIncomingRequests(); +// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", +// Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) { +// updateSeedNodes(connection, Arrays.asList(seedNode)); +// assertTrue(service.nodeConnected(seedNode)); +// assertTrue(service.nodeConnected(discoverableNode)); +// assertFalse(service.nodeConnected(spareNode)); +// knownNodes.add(spareNode); +// CountDownLatch latchDisconnect = new CountDownLatch(1); +// CountDownLatch latchConnected = new CountDownLatch(1); +// service.addNodeConnectionListener(new TransportConnectionListener() { +// @Override +// public void onNodeDisconnected(DiscoveryNode node) { +// if (node.equals(discoverableNode)) { +// latchDisconnect.countDown(); +// } +// } +// +// @Override +// public void onNodeConnected(DiscoveryNode node) { +// if (node.equals(spareNode)) { +// latchConnected.countDown(); +// } +// } +// }); +// +// discoverableTransport.close(); +// // now make sure we try to connect again to other nodes once we got disconnected +// assertTrue(latchDisconnect.await(10, TimeUnit.SECONDS)); +// assertTrue(latchConnected.await(10, TimeUnit.SECONDS)); +// assertTrue(service.nodeConnected(spareNode)); +// } +// } +// } +// } public void testFilterDiscoveredNodes() throws Exception { List knownNodes = new CopyOnWriteArrayList<>(); @@ -1296,7 +1296,7 @@ public void close() { // } // return super.getConnection(node); // } - +// // @Override // public boolean nodeConnected(DiscoveryNode node) { // return node.equals(connectedNode); From f699321f2203caf29f92ec9d1ae29efcbd79bb19 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jul 2018 18:44:56 -0600 Subject: [PATCH 09/29] Remove unused --- .../transport/TransportService.java | 7 ++-- .../test/transport/MockConnectionManager.java | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index f1bb283168deb..c0cfb8ea03f0c 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -21,12 +21,10 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.elasticsearch.action.termvectors.TermVectorsFilter; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; @@ -48,6 +46,7 @@ import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskManager; diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java new file mode 100644 index 0000000000000..898bc0b156b2d --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test.transport; + +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ConnectionManager; +import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.transport.Transport; + +public class MockConnectionManager extends ConnectionManager { + + public MockConnectionManager(Settings settings, Transport transport, ThreadPool threadPool) { + super(Loggers.getLogger(MockConnectionManager.class, settings), transport, threadPool, TcpTransport.PING_SCHEDULE.get(settings)); + } + +} From 03acc1979028687b4fa43be9cb53dc924ec0338f Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jul 2018 22:12:51 -0600 Subject: [PATCH 10/29] Fix checkstyle --- .../cluster/NodeConnectionsServiceTests.java | 17 ++-- .../RemoteClusterConnectionTests.java | 98 +++++++++---------- 2 files changed, 58 insertions(+), 57 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index d64c903c3442b..6f138aadd08e2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -149,7 +149,8 @@ public void setUp() throws Exception { super.setUp(); this.threadPool = new TestThreadPool(getClass().getName()); this.transport = new MockTransport(); - transportService = new NoHandshakeTransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, + transportService = new NoHandshakeTransportService(Settings.EMPTY, transport, threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), UUIDs.randomBase64UUID()), null, Collections.emptySet()); transportService.start(); @@ -167,13 +168,13 @@ public void tearDown() throws Exception { private final class NoHandshakeTransportService extends TransportService { - public NoHandshakeTransportService(Settings settings, - Transport transport, - ThreadPool threadPool, - TransportInterceptor transportInterceptor, - Function localNodeFactory, - ClusterSettings clusterSettings, - Set taskHeaders) { + private NoHandshakeTransportService(Settings settings, + Transport transport, + ThreadPool threadPool, + TransportInterceptor transportInterceptor, + Function localNodeFactory, + ClusterSettings clusterSettings, + Set taskHeaders) { super(settings, transport, threadPool, transportInterceptor, localNodeFactory, clusterSettings, taskHeaders); } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 22ae194ede056..a2432d5f79c03 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -294,55 +294,55 @@ public void testDiscoverSingleNodeWithIncompatibleSeed() throws Exception { } } -// public void testNodeDisconnected() throws Exception { -// List knownNodes = new CopyOnWriteArrayList<>(); -// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); -// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT); -// MockTransportService spareTransport = startTransport("spare_node", knownNodes, Version.CURRENT)) { -// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); -// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); -// DiscoveryNode spareNode = spareTransport.getLocalDiscoNode(); -// knownNodes.add(seedTransport.getLocalDiscoNode()); -// knownNodes.add(discoverableTransport.getLocalDiscoNode()); -// Collections.shuffle(knownNodes, random()); -// -// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { -// service.start(); -// service.acceptIncomingRequests(); -// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", -// Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) { -// updateSeedNodes(connection, Arrays.asList(seedNode)); -// assertTrue(service.nodeConnected(seedNode)); -// assertTrue(service.nodeConnected(discoverableNode)); -// assertFalse(service.nodeConnected(spareNode)); -// knownNodes.add(spareNode); -// CountDownLatch latchDisconnect = new CountDownLatch(1); -// CountDownLatch latchConnected = new CountDownLatch(1); -// service.addNodeConnectionListener(new TransportConnectionListener() { -// @Override -// public void onNodeDisconnected(DiscoveryNode node) { -// if (node.equals(discoverableNode)) { -// latchDisconnect.countDown(); -// } -// } -// -// @Override -// public void onNodeConnected(DiscoveryNode node) { -// if (node.equals(spareNode)) { -// latchConnected.countDown(); -// } -// } -// }); -// -// discoverableTransport.close(); -// // now make sure we try to connect again to other nodes once we got disconnected -// assertTrue(latchDisconnect.await(10, TimeUnit.SECONDS)); -// assertTrue(latchConnected.await(10, TimeUnit.SECONDS)); -// assertTrue(service.nodeConnected(spareNode)); -// } -// } -// } -// } + public void testNodeDisconnected() throws Exception { + List knownNodes = new CopyOnWriteArrayList<>(); + try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); + MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT); + MockTransportService spareTransport = startTransport("spare_node", knownNodes, Version.CURRENT)) { + DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); + DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); + DiscoveryNode spareNode = spareTransport.getLocalDiscoNode(); + knownNodes.add(seedTransport.getLocalDiscoNode()); + knownNodes.add(discoverableTransport.getLocalDiscoNode()); + Collections.shuffle(knownNodes, random()); + + try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + service.start(); + service.acceptIncomingRequests(); + try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", + Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) { + updateSeedNodes(connection, Arrays.asList(seedNode)); + assertTrue(service.nodeConnected(seedNode)); + assertTrue(service.nodeConnected(discoverableNode)); + assertFalse(service.nodeConnected(spareNode)); + knownNodes.add(spareNode); + CountDownLatch latchDisconnect = new CountDownLatch(1); + CountDownLatch latchConnected = new CountDownLatch(1); + service.addNodeConnectionListener(new TransportConnectionListener() { + @Override + public void onNodeDisconnected(DiscoveryNode node) { + if (node.equals(discoverableNode)) { + latchDisconnect.countDown(); + } + } + + @Override + public void onNodeConnected(DiscoveryNode node) { + if (node.equals(spareNode)) { + latchConnected.countDown(); + } + } + }); + + discoverableTransport.close(); + // now make sure we try to connect again to other nodes once we got disconnected + assertTrue(latchDisconnect.await(10, TimeUnit.SECONDS)); + assertTrue(latchConnected.await(10, TimeUnit.SECONDS)); + assertTrue(service.nodeConnected(spareNode)); + } + } + } + } public void testFilterDiscoveredNodes() throws Exception { List knownNodes = new CopyOnWriteArrayList<>(); From 796a728598a058bebf0dd34e2b7c0fe5dd6f072f Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jul 2018 17:02:18 -0600 Subject: [PATCH 11/29] Work on test infra --- .../transport/ConnectionManager.java | 20 +-- .../elasticsearch/transport/TcpTransport.java | 3 +- .../transport/TransportService.java | 9 +- .../test/transport/MockConnectionManager.java | 110 ++++++++++++- .../test/transport/MockTransportService.java | 152 ++++++++++++++---- 5 files changed, 245 insertions(+), 49 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index c40bd7cb13ad9..7906efaabae14 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -25,6 +25,8 @@ import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; @@ -53,11 +55,11 @@ public class ConnectionManager implements Closeable { private final Lifecycle lifecycle = new Lifecycle(); private final DelegatingNodeConnectionListener connectionListener = new DelegatingNodeConnectionListener(); - public ConnectionManager(Logger logger, Transport transport, ThreadPool threadPool, TimeValue pingSchedule) { - this.logger = logger; + public ConnectionManager(Settings settings, Transport transport, ThreadPool threadPool) { + this.logger = Loggers.getLogger(getClass(), settings); this.transport = transport; this.threadPool = threadPool; - this.pingSchedule = pingSchedule; + this.pingSchedule = TcpTransport.PING_SCHEDULE.get(settings); this.lifecycle.moveToStarted(); if (pingSchedule.millis() > 0) { @@ -141,10 +143,8 @@ public void disconnectFromNode(DiscoveryNode node) { } } - private void ensureOpen() { - if (lifecycle.started() == false) { - throw new IllegalStateException("connection manager is closed"); - } + public int connectedNodeCount() { + return connectedNodes.size(); } @Override @@ -167,8 +167,10 @@ public void close() { lifecycle.moveToClosed(); } - public int connectedNodeCount() { - return connectedNodes.size(); + private void ensureOpen() { + if (lifecycle.started() == false) { + throw new IllegalStateException("connection manager is closed"); + } } private class ScheduledPing extends AbstractLifecycleRunnable { diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index 5868c0d968eaa..ef3d228384e4e 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -941,8 +941,7 @@ public boolean canCompress(TransportRequest request) { private void sendRequestToChannel(final DiscoveryNode node, final TcpChannel channel, final long requestId, final String action, final TransportRequest request, TransportRequestOptions options, Version channelVersion, - byte status) throws IOException, - TransportException { + byte status) throws IOException, TransportException { if (compress) { options = TransportRequestOptions.builder(options).withCompress(true).build(); } diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index c0cfb8ea03f0c..3f4d57d9c8b22 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -147,6 +147,13 @@ public void close() { public TransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor transportInterceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, Set taskHeaders) { + this(settings, transport, threadPool, transportInterceptor, localNodeFactory, clusterSettings, taskHeaders, + new ConnectionManager(settings, transport, threadPool)); + } + + public TransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor transportInterceptor, + Function localNodeFactory, @Nullable ClusterSettings clusterSettings, + Set taskHeaders, ConnectionManager connectionManager) { super(settings); // The only time we do not want to validate node connections is when this is a transport client using the simple node sampler this.validateConnections = TransportClient.CLIENT_TYPE.equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey())) == false || @@ -154,7 +161,7 @@ public TransportService(Settings settings, Transport transport, ThreadPool threa this.transport = transport; this.threadPool = threadPool; this.localNodeFactory = localNodeFactory; - this.connectionManager = new ConnectionManager(logger, transport, threadPool, TcpTransport.PING_SCHEDULE.get(settings)); + this.connectionManager = connectionManager; this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings); setTracerLogInclude(TRACE_LOG_INCLUDE_SETTING.get(settings)); setTracerLogExclude(TRACE_LOG_EXCLUDE_SETTING.get(settings)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java index 898bc0b156b2d..13c30b5300fc6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java @@ -18,17 +18,119 @@ */ package org.elasticsearch.test.transport; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectionManager; -import org.elasticsearch.transport.TcpTransport; import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; public class MockConnectionManager extends ConnectionManager { - public MockConnectionManager(Settings settings, Transport transport, ThreadPool threadPool) { - super(Loggers.getLogger(MockConnectionManager.class, settings), transport, threadPool, TcpTransport.PING_SCHEDULE.get(settings)); + private final ConcurrentHashMap sendBehaviors = new ConcurrentHashMap<>(); + private volatile MockTransportService.SendRequestBehavior defaultSendRequest = null; + + MockConnectionManager(Settings settings, Transport transport, ThreadPool threadPool) { + super(settings, transport, threadPool); + } + + @Override + public Transport.Connection getConnection(DiscoveryNode node) { + return new WrappedConnection(super.getConnection(node)); } + public void addSendBehavior(TransportAddress transportAddress, MockTransportService.SendRequestBehavior sendBehavior) { + sendBehaviors.put(transportAddress, sendBehavior); + } + + public void clearSendBehaviors() { + sendBehaviors.clear(); + } + + public void clearSendBehavior(TransportAddress transportAddress) { + MockTransportService.SendRequestBehavior behavior = sendBehaviors.remove(transportAddress); + if (behavior != null) { + behavior.clearCallback(); + } + } + + public Transport.Connection wrapWithBehavior(Transport.Connection connection) { + return new WrappedConnection(connection); + } + + private class WrappedConnection implements Transport.Connection { + + private final Transport.Connection connection; + + private WrappedConnection(Transport.Connection connection) { + this.connection = connection; + } + + @Override + public DiscoveryNode getNode() { + return connection.getNode(); + } + + @Override + public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) + throws IOException, TransportException { + TransportAddress address = connection.getNode().getAddress(); + MockTransportService.SendRequestBehavior behavior = sendBehaviors.getOrDefault(address, defaultSendRequest); + if (behavior == null) { + connection.sendRequest(requestId, action, request, options); + } else { + // TODO: Connections maybe not the same? + behavior.sendRequest(this, requestId, action, request, options); + } + } + + @Override + public boolean sendPing() { + return connection.sendPing(); + } + + @Override + public void addCloseListener(ActionListener listener) { + connection.addCloseListener(listener); + } + + + @Override + public boolean isClosed() { + return connection.isClosed(); + } + + @Override + public Version getVersion() { + return getNode().getVersion(); + } + + @Override + public Object getCacheKey() { + return connection.getCacheKey(); + } + + @Override + public void close() throws IOException { + connection.close(); + } + + @Override + public boolean equals(Object obj) { + return connection.equals(obj); + } + + @Override + public int hashCode() { + return connection.hashCode(); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 85a3c1831d28c..94bbacc786990 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -74,6 +74,7 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; +import java.util.function.Supplier; /** * A mock transport service that allows to simulate different network topology failures. @@ -122,6 +123,7 @@ public static MockTransportService createNewService(Settings settings, Transport } private final Transport original; + private final MockConnectionManager connectionManager; /** * Build the service. @@ -145,9 +147,18 @@ public MockTransportService(Settings settings, Transport transport, ThreadPool t public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, Set taskHeaders) { + this(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders, + new MockConnectionManager(settings, transport, threadPool)); + } + + private MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor, + Function localNodeFactory, + @Nullable ClusterSettings clusterSettings, Set taskHeaders, + MockConnectionManager connectionManager) { super(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, - taskHeaders); + taskHeaders, new MockConnectionManager(settings, transport, threadPool)); this.original = transport; + this.connectionManager = connectionManager; } public static TransportAddress[] extractTransportAddresses(TransportService transportService) { @@ -171,6 +182,7 @@ protected TaskManager createTaskManager(Settings settings, ThreadPool threadPool * Clears all the registered rules. */ public void clearAllRules() { + connectionManager.clearSendBehaviors(); transport().transports.clear(); } @@ -222,13 +234,19 @@ public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throw new ConnectTransportException(node, "DISCONNECT: simulated"); } - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { - connection.close(); - // send the request, which will blow up - connection.sendRequest(requestId, action, request, options); - } +// @Override +// protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, +// TransportRequestOptions options) throws IOException { +// connection.close(); +// // send the request, which will blow up +// connection.sendRequest(requestId, action, request, options); +// } + }); + + connectionManager.addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { + connection.close(); + // send the request, which will blow up + connection.sendRequest(requestId, action, request, options); }); } @@ -259,18 +277,25 @@ public void addFailToSendNoConnectRule(TransportService transportService, final * Adds a rule that will cause matching operations to throw ConnectTransportExceptions */ public void addFailToSendNoConnectRule(TransportAddress transportAddress, final Set blockedActions) { - - addDelegate(transportAddress, new DelegateTransport(original) { - - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { - if (blockedActions.contains(action)) { - logger.info("--> preventing {} request", action); - connection.close(); - } - connection.sendRequest(requestId, action, request, options); +// addDelegate(transportAddress, new DelegateTransport(original) { +// +// @Override +// protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, +// TransportRequestOptions options) throws IOException { +// if (blockedActions.contains(action)) { +// logger.info("--> preventing {} request", action); +// connection.close(); +// } +// connection.sendRequest(requestId, action, request, options); +// } +// }); + + connectionManager.addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { + if (blockedActions.contains(action)) { + logger.info("--> preventing {} request", action); + connection.close(); } + connection.sendRequest(requestId, action, request, options); }); } @@ -295,12 +320,10 @@ public void addUnresponsiveRule(TransportAddress transportAddress) { public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); } + }); - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { - // don't send anything, the receiving node is unresponsive - } + connectionManager.addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { + // don't send anything, the receiving node is unresponsive }); } @@ -325,17 +348,15 @@ public void addUnresponsiveRule(TransportService transportService, final TimeVal public void addUnresponsiveRule(TransportAddress transportAddress, final TimeValue duration) { final long startTime = System.currentTimeMillis(); + Supplier delaySupplier = () -> new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime)); + addDelegate(transportAddress, new ClearableTransport(original) { private final Queue requestsToSendWhenCleared = new LinkedBlockingDeque<>(); private boolean cleared = false; - TimeValue getDelay() { - return new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime)); - } - @Override public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - TimeValue delay = getDelay(); + TimeValue delay = delaySupplier.get(); if (delay.millis() <= 0) { return original.openConnection(node, profile); } @@ -355,11 +376,68 @@ public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) } } +// @Override +// protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, +// TransportRequestOptions options) throws IOException { +// // delayed sending - even if larger then the request timeout to simulated a potential late response from target node +// TimeValue delay = getDelay(); +// if (delay.millis() <= 0) { +// connection.sendRequest(requestId, action, request, options); +// return; +// } +// +// // poor mans request cloning... +// RequestHandlerRegistry reg = MockTransportService.this.getRequestHandler(action); +// BytesStreamOutput bStream = new BytesStreamOutput(); +// request.writeTo(bStream); +// final TransportRequest clonedRequest = reg.newRequest(bStream.bytes().streamInput()); +// +// Runnable runnable = new AbstractRunnable() { +// AtomicBoolean requestSent = new AtomicBoolean(); +// +// @Override +// public void onFailure(Exception e) { +// logger.debug("failed to send delayed request", e); +// } +// +// @Override +// protected void doRun() throws IOException { +// if (requestSent.compareAndSet(false, true)) { +// connection.sendRequest(requestId, action, clonedRequest, options); +// } +// } +// }; +// +// // store the request to send it once the rule is cleared. +// synchronized (this) { +// if (cleared) { +// runnable.run(); +// } else { +// requestsToSendWhenCleared.add(runnable); +// threadPool.schedule(delay, ThreadPool.Names.GENERIC, runnable); +// } +// } +// } + +// @Override + public void clearRule() { +// synchronized (this) { +// assert cleared == false; +// cleared = true; +// requestsToSendWhenCleared.forEach(Runnable::run); +// } + } + }); + + connectionManager.addSendBehavior(transportAddress, new SendRequestBehavior() { + private final Queue requestsToSendWhenCleared = new LinkedBlockingDeque<>(); + private boolean cleared = false; + @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { + public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, + TransportRequestOptions options) throws IOException { // delayed sending - even if larger then the request timeout to simulated a potential late response from target node - TimeValue delay = getDelay(); + TimeValue delay = delaySupplier.get(); if (delay.millis() <= 0) { connection.sendRequest(requestId, action, request, options); return; @@ -399,7 +477,7 @@ protected void doRun() throws IOException { } @Override - public void clearRule() { + public void clearCallback() { synchronized (this) { assert cleared == false; cleared = true; @@ -750,4 +828,12 @@ protected void doClose() throws IOException { public DiscoveryNode getLocalDiscoNode() { return this.getLocalNode(); } + + @FunctionalInterface + public interface SendRequestBehavior { + void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, + TransportRequestOptions options) throws IOException; + + default void clearCallback() {}; + } } From 43dd3ada74b0889a935f6be47f542e63c8336bd2 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jul 2018 18:32:05 -0600 Subject: [PATCH 12/29] Work on tests --- .../transport/RemoteClusterConnection.java | 1 + .../RemoteClusterConnectionTests.java | 8 +- .../test/transport/MockConnectionManager.java | 136 ---------- .../test/transport/MockTransportService.java | 233 ++++++++---------- 4 files changed, 107 insertions(+), 271 deletions(-) delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 355a9c655c998..5c23ea227d574 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -122,6 +122,7 @@ final class RemoteClusterConnection extends AbstractComponent implements Transpo .getConcreteSettingForNamespace(clusterAlias).get(settings); this.connectHandler = new ConnectHandler(); transportService.addConnectionListener(this); + transportService.addNodeConnectionListener(this); } /** diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index a2432d5f79c03..cdf663fa353e5 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -440,7 +440,7 @@ public void close() throws IOException { // no-op } }; - service.addDelegate(seedNode.getAddress(), new MockTransportService.DelegateTransport(service.getOriginalTransport()) { + service.addDelegate(seedNode.getAddress(), new MockTransportService.DelegateTransport(service.original()) { // @Override // public Connection getConnection(DiscoveryNode node) { // if (node == seedNode) { @@ -664,7 +664,9 @@ public void onNodeDisconnected(DiscoveryNode node) { AtomicReference reference = new AtomicReference<>(); AtomicReference failReference = new AtomicReference<>(); connection.fetchSearchShards(searchShardsRequest, - new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch)); + new LatchedActionListener<>(ActionListener.wrap((s) -> { + reference.set(s); + }, failReference::set), responseLatch)); assertTrue(responseLatch.await(1, TimeUnit.SECONDS)); assertNotNull(failReference.get()); assertNull(reference.get()); @@ -1288,7 +1290,7 @@ public void close() { // no-op } }; - service.addDelegate(connectedNode.getAddress(), new MockTransportService.DelegateTransport(service.getOriginalTransport()) { + service.addDelegate(connectedNode.getAddress(), new MockTransportService.DelegateTransport(service.original()) { // @Override // public Connection getConnection(DiscoveryNode node) { // if (node == connectedNode) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java deleted file mode 100644 index 13c30b5300fc6..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.transport; - -import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.ConnectionManager; -import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportException; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; - -import java.io.IOException; -import java.util.concurrent.ConcurrentHashMap; - -public class MockConnectionManager extends ConnectionManager { - - private final ConcurrentHashMap sendBehaviors = new ConcurrentHashMap<>(); - private volatile MockTransportService.SendRequestBehavior defaultSendRequest = null; - - MockConnectionManager(Settings settings, Transport transport, ThreadPool threadPool) { - super(settings, transport, threadPool); - } - - @Override - public Transport.Connection getConnection(DiscoveryNode node) { - return new WrappedConnection(super.getConnection(node)); - } - - public void addSendBehavior(TransportAddress transportAddress, MockTransportService.SendRequestBehavior sendBehavior) { - sendBehaviors.put(transportAddress, sendBehavior); - } - - public void clearSendBehaviors() { - sendBehaviors.clear(); - } - - public void clearSendBehavior(TransportAddress transportAddress) { - MockTransportService.SendRequestBehavior behavior = sendBehaviors.remove(transportAddress); - if (behavior != null) { - behavior.clearCallback(); - } - } - - public Transport.Connection wrapWithBehavior(Transport.Connection connection) { - return new WrappedConnection(connection); - } - - private class WrappedConnection implements Transport.Connection { - - private final Transport.Connection connection; - - private WrappedConnection(Transport.Connection connection) { - this.connection = connection; - } - - @Override - public DiscoveryNode getNode() { - return connection.getNode(); - } - - @Override - public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws IOException, TransportException { - TransportAddress address = connection.getNode().getAddress(); - MockTransportService.SendRequestBehavior behavior = sendBehaviors.getOrDefault(address, defaultSendRequest); - if (behavior == null) { - connection.sendRequest(requestId, action, request, options); - } else { - // TODO: Connections maybe not the same? - behavior.sendRequest(this, requestId, action, request, options); - } - } - - @Override - public boolean sendPing() { - return connection.sendPing(); - } - - @Override - public void addCloseListener(ActionListener listener) { - connection.addCloseListener(listener); - } - - - @Override - public boolean isClosed() { - return connection.isClosed(); - } - - @Override - public Version getVersion() { - return getNode().getVersion(); - } - - @Override - public Object getCacheKey() { - return connection.getCacheKey(); - } - - @Override - public void close() throws IOException { - connection.close(); - } - - @Override - public boolean equals(Object obj) { - return connection.equals(obj); - } - - @Override - public int hashCode() { - return connection.hashCode(); - } - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 94bbacc786990..f2235b0adc32d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -21,6 +21,7 @@ import com.carrotsearch.randomizedtesting.SysGlobals; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; @@ -46,6 +47,7 @@ import org.elasticsearch.test.tasks.MockTaskManager; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.ConnectionManager; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.MockTcpTransport; import org.elasticsearch.transport.RequestHandlerRegistry; @@ -69,6 +71,7 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.LinkedBlockingDeque; @@ -123,7 +126,6 @@ public static MockTransportService createNewService(Settings settings, Transport } private final Transport original; - private final MockConnectionManager connectionManager; /** * Build the service. @@ -147,18 +149,8 @@ public MockTransportService(Settings settings, Transport transport, ThreadPool t public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, Set taskHeaders) { - this(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders, - new MockConnectionManager(settings, transport, threadPool)); - } - - private MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor, - Function localNodeFactory, - @Nullable ClusterSettings clusterSettings, Set taskHeaders, - MockConnectionManager connectionManager) { - super(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, - taskHeaders, new MockConnectionManager(settings, transport, threadPool)); + super(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); this.original = transport; - this.connectionManager = connectionManager; } public static TransportAddress[] extractTransportAddresses(TransportService transportService) { @@ -182,7 +174,7 @@ protected TaskManager createTaskManager(Settings settings, ThreadPool threadPool * Clears all the registered rules. */ public void clearAllRules() { - connectionManager.clearSendBehaviors(); + transport().clearSendBehaviors(); transport().transports.clear(); } @@ -199,10 +191,8 @@ public void clearRule(TransportService transportService) { * Clears the rule associated with the provided transport address. */ public void clearRule(TransportAddress transportAddress) { - Transport transport = transport().transports.remove(transportAddress); - if (transport instanceof ClearableTransport) { - ((ClearableTransport) transport).clearRule(); - } + transport().transports.remove(transportAddress); + transport().clearSendBehavior(transportAddress); } /** @@ -233,17 +223,9 @@ public void addFailToSendNoConnectRule(TransportAddress transportAddress) { public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { throw new ConnectTransportException(node, "DISCONNECT: simulated"); } - -// @Override -// protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, -// TransportRequestOptions options) throws IOException { -// connection.close(); -// // send the request, which will blow up -// connection.sendRequest(requestId, action, request, options); -// } }); - connectionManager.addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { + transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { connection.close(); // send the request, which will blow up connection.sendRequest(requestId, action, request, options); @@ -277,20 +259,7 @@ public void addFailToSendNoConnectRule(TransportService transportService, final * Adds a rule that will cause matching operations to throw ConnectTransportExceptions */ public void addFailToSendNoConnectRule(TransportAddress transportAddress, final Set blockedActions) { -// addDelegate(transportAddress, new DelegateTransport(original) { -// -// @Override -// protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, -// TransportRequestOptions options) throws IOException { -// if (blockedActions.contains(action)) { -// logger.info("--> preventing {} request", action); -// connection.close(); -// } -// connection.sendRequest(requestId, action, request, options); -// } -// }); - - connectionManager.addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { + transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { if (blockedActions.contains(action)) { logger.info("--> preventing {} request", action); connection.close(); @@ -322,7 +291,7 @@ public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) } }); - connectionManager.addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { + transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { // don't send anything, the receiving node is unresponsive }); } @@ -350,9 +319,7 @@ public void addUnresponsiveRule(TransportAddress transportAddress, final TimeVal Supplier delaySupplier = () -> new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime)); - addDelegate(transportAddress, new ClearableTransport(original) { - private final Queue requestsToSendWhenCleared = new LinkedBlockingDeque<>(); - private boolean cleared = false; + addDelegate(transportAddress, new DelegateTransport(original) { @Override public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { @@ -375,61 +342,9 @@ public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); } } - -// @Override -// protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, -// TransportRequestOptions options) throws IOException { -// // delayed sending - even if larger then the request timeout to simulated a potential late response from target node -// TimeValue delay = getDelay(); -// if (delay.millis() <= 0) { -// connection.sendRequest(requestId, action, request, options); -// return; -// } -// -// // poor mans request cloning... -// RequestHandlerRegistry reg = MockTransportService.this.getRequestHandler(action); -// BytesStreamOutput bStream = new BytesStreamOutput(); -// request.writeTo(bStream); -// final TransportRequest clonedRequest = reg.newRequest(bStream.bytes().streamInput()); -// -// Runnable runnable = new AbstractRunnable() { -// AtomicBoolean requestSent = new AtomicBoolean(); -// -// @Override -// public void onFailure(Exception e) { -// logger.debug("failed to send delayed request", e); -// } -// -// @Override -// protected void doRun() throws IOException { -// if (requestSent.compareAndSet(false, true)) { -// connection.sendRequest(requestId, action, clonedRequest, options); -// } -// } -// }; -// -// // store the request to send it once the rule is cleared. -// synchronized (this) { -// if (cleared) { -// runnable.run(); -// } else { -// requestsToSendWhenCleared.add(runnable); -// threadPool.schedule(delay, ThreadPool.Names.GENERIC, runnable); -// } -// } -// } - -// @Override - public void clearRule() { -// synchronized (this) { -// assert cleared == false; -// cleared = true; -// requestsToSendWhenCleared.forEach(Runnable::run); -// } - } }); - connectionManager.addSendBehavior(transportAddress, new SendRequestBehavior() { + transport().addSendBehavior(transportAddress, new SendRequestBehavior() { private final Queue requestsToSendWhenCleared = new LinkedBlockingDeque<>(); private boolean cleared = false; @@ -519,23 +434,101 @@ private LookupTestTransport transport() { */ private static class LookupTestTransport extends DelegateTransport { - final ConcurrentMap transports = ConcurrentCollections.newConcurrentMap(); + private final ConcurrentMap transports = ConcurrentCollections.newConcurrentMap(); + private final ConcurrentHashMap sendBehaviors = new ConcurrentHashMap<>(); + private volatile MockTransportService.SendRequestBehavior defaultSendRequest = null; - LookupTestTransport(Transport transport) { + private LookupTestTransport(Transport transport) { super(transport); } - private Transport getTransport(DiscoveryNode node) { - Transport transport = transports.get(node.getAddress()); - if (transport != null) { - return transport; + @Override + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { + Transport transportToUse = transports.getOrDefault(node.getAddress(), transport); + return new WrappedConnection(transportToUse.openConnection(node, profile)); + } + + void addSendBehavior(TransportAddress transportAddress, MockTransportService.SendRequestBehavior sendBehavior) { + sendBehaviors.put(transportAddress, sendBehavior); + } + + void clearSendBehaviors() { + sendBehaviors.clear(); + } + + void clearSendBehavior(TransportAddress transportAddress) { + MockTransportService.SendRequestBehavior behavior = sendBehaviors.remove(transportAddress); + if (behavior != null) { + behavior.clearCallback(); } - return this.transport; } - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - return getTransport(node).openConnection(node, profile); + private class WrappedConnection implements Transport.Connection { + + private final Transport.Connection connection; + + private WrappedConnection(Transport.Connection connection) { + this.connection = connection; + } + + @Override + public DiscoveryNode getNode() { + return connection.getNode(); + } + + @Override + public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) + throws IOException, TransportException { + TransportAddress address = connection.getNode().getAddress(); + MockTransportService.SendRequestBehavior behavior = sendBehaviors.getOrDefault(address, defaultSendRequest); + if (behavior == null) { + connection.sendRequest(requestId, action, request, options); + } else { + // TODO: Connections maybe not the same? + behavior.sendRequest(connection, requestId, action, request, options); + } + } + + @Override + public boolean sendPing() { + return connection.sendPing(); + } + + @Override + public void addCloseListener(ActionListener listener) { + connection.addCloseListener(listener); + } + + + @Override + public boolean isClosed() { + return connection.isClosed(); + } + + @Override + public Version getVersion() { + return getNode().getVersion(); + } + + @Override + public Object getCacheKey() { + return connection.getCacheKey(); + } + + @Override + public void close() throws IOException { + connection.close(); + } + + @Override + public boolean equals(Object obj) { + return connection.equals(obj); + } + + @Override + public int hashCode() { + return connection.hashCode(); + } } } @@ -589,13 +582,7 @@ public List getLocalAddresses() { @Override public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - return new FilteredConnection(transport.openConnection(node, profile)) { - @Override - public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws IOException, TransportException { - DelegateTransport.this.sendRequest(connection, requestId, action, request, options); - } - }; + return transport.openConnection(node, profile); } @Override @@ -649,24 +636,6 @@ protected void sendRequest(Transport.Connection connection, long requestId, Stri } } - /** - * The delegate transport instances defined in this class mock various kinds of disruption types. This subclass adds a method - * {@link #clearRule()} so that when the disruptions are cleared (see {@link #clearRule(TransportService)}) this gives the - * disruption a possibility to run clean-up actions. - */ - public abstract static class ClearableTransport extends DelegateTransport { - - public ClearableTransport(Transport transport) { - super(transport); - } - - /** - * Called by {@link #clearRule(TransportService)} - */ - public abstract void clearRule(); - } - - List activeTracers = new CopyOnWriteArrayList<>(); public static class Tracer { From acb743a35545f0fd4d817c0d6e04b6f9041d6aec Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jul 2018 18:34:50 -0600 Subject: [PATCH 13/29] Remove methods --- .../transport/RemoteClusterConnection.java | 1 - .../transport/TransportService.java | 10 ++------- .../discovery/ZenFaultDetectionTests.java | 8 +++---- .../zen/PublishClusterStateActionTests.java | 8 +++---- .../transport/RemoteClusterClientTests.java | 2 +- .../RemoteClusterConnectionTests.java | 4 ++-- .../transport/RemoteClusterServiceTests.java | 2 +- .../AbstractSimpleTransportTestCase.java | 22 +++++++++---------- 8 files changed, 25 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 5c23ea227d574..355a9c655c998 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -122,7 +122,6 @@ final class RemoteClusterConnection extends AbstractComponent implements Transpo .getConcreteSettingForNamespace(clusterAlias).get(settings); this.connectHandler = new ConnectHandler(); transportService.addConnectionListener(this); - transportService.addNodeConnectionListener(this); } /** diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 3f4d57d9c8b22..5ddd61b5040a3 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -482,20 +482,14 @@ public void disconnectFromNode(DiscoveryNode node) { connectionManager.disconnectFromNode(node); } - public void addNodeConnectionListener(TransportConnectionListener listener) { - connectionManager.addListener(listener); - } - - public void removeNodeConnectionListener(TransportConnectionListener listener) { - connectionManager.removeListener(listener); - } - public void addConnectionListener(TransportConnectionListener listener) { transport.addConnectionListener(listener); + connectionManager.addListener(listener); } public void removeConnectionListener(TransportConnectionListener listener) { transport.removeConnectionListener(listener); + connectionManager.removeListener(listener); } public TransportFuture submitRequest(DiscoveryNode node, String action, TransportRequest request, diff --git a/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java index b573ecd6ef2d5..03c0df43591ba 100644 --- a/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java @@ -107,8 +107,8 @@ public void onNodeDisconnected(DiscoveryNode node) { fail("disconnect should not be called " + node); } }; - serviceA.addNodeConnectionListener(waitForConnection); - serviceB.addNodeConnectionListener(waitForConnection); + serviceA.addConnectionListener(waitForConnection); + serviceB.addConnectionListener(waitForConnection); serviceA.connectToNode(nodeB); serviceA.connectToNode(nodeA); @@ -116,8 +116,8 @@ public void onNodeDisconnected(DiscoveryNode node) { serviceB.connectToNode(nodeB); assertThat("failed to wait for all nodes to connect", latch.await(5, TimeUnit.SECONDS), equalTo(true)); - serviceA.removeNodeConnectionListener(waitForConnection); - serviceB.removeNodeConnectionListener(waitForConnection); + serviceA.removeConnectionListener(waitForConnection); + serviceB.removeConnectionListener(waitForConnection); } @Override diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java index c9a634c7f9ce7..ac1719269e7ae 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java @@ -201,17 +201,17 @@ public void onNodeDisconnected(DiscoveryNode node) { fail("disconnect should not be called " + node); } }; - node.service.addNodeConnectionListener(waitForConnection); + node.service.addConnectionListener(waitForConnection); for (MockNode curNode : nodes.values()) { - curNode.service.addNodeConnectionListener(waitForConnection); + curNode.service.addConnectionListener(waitForConnection); curNode.connectTo(node.discoveryNode); node.connectTo(curNode.discoveryNode); } assertThat("failed to wait for all nodes to connect", latch.await(5, TimeUnit.SECONDS), equalTo(true)); for (MockNode curNode : nodes.values()) { - curNode.service.removeNodeConnectionListener(waitForConnection); + curNode.service.removeConnectionListener(waitForConnection); } - node.service.removeNodeConnectionListener(waitForConnection); + node.service.removeConnectionListener(waitForConnection); if (nodes.put(name, node) != null) { fail("Node with the name " + name + " already exist"); } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java index 9c81896624d63..8cfec0a07f910 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java @@ -81,7 +81,7 @@ public void testEnsureWeReconnect() throws Exception { try (MockTransportService service = MockTransportService.createNewService(localSettings, Version.CURRENT, threadPool, null)) { Semaphore semaphore = new Semaphore(1); service.start(); - service.addNodeConnectionListener(new TransportConnectionListener() { + service.addConnectionListener(new TransportConnectionListener() { @Override public void onNodeDisconnected(DiscoveryNode node) { if (remoteNode.equals(node)) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index cdf663fa353e5..15d1817f832af 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -318,7 +318,7 @@ public void testNodeDisconnected() throws Exception { knownNodes.add(spareNode); CountDownLatch latchDisconnect = new CountDownLatch(1); CountDownLatch latchConnected = new CountDownLatch(1); - service.addNodeConnectionListener(new TransportConnectionListener() { + service.addConnectionListener(new TransportConnectionListener() { @Override public void onNodeDisconnected(DiscoveryNode node) { if (node.equals(discoverableNode)) { @@ -645,7 +645,7 @@ public void testFetchShardsSkipUnavailable() throws Exception { } CountDownLatch disconnectedLatch = new CountDownLatch(1); - service.addNodeConnectionListener(new TransportConnectionListener() { + service.addConnectionListener(new TransportConnectionListener() { @Override public void onNodeDisconnected(DiscoveryNode node) { if (node.equals(seedNode)) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java index aa4ed509952cc..03d76b5a953c6 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java @@ -584,7 +584,7 @@ public void testCollectSearchShards() throws Exception { } CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters); - service.addNodeConnectionListener(new TransportConnectionListener() { + service.addConnectionListener(new TransportConnectionListener() { @Override public void onNodeDisconnected(DiscoveryNode node) { if (disconnectedNodes.remove(node)) { diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index b821d36364d56..50b7b8ce57597 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -139,8 +139,8 @@ public void onNodeDisconnected(DiscoveryNode node) { fail("disconnect should not be called " + node); } }; - serviceA.addNodeConnectionListener(waitForConnection); - serviceB.addNodeConnectionListener(waitForConnection); + serviceA.addConnectionListener(waitForConnection); + serviceB.addConnectionListener(waitForConnection); int numHandshakes = 1; serviceA.connectToNode(nodeB); serviceB.connectToNode(nodeA); @@ -148,8 +148,8 @@ public void onNodeDisconnected(DiscoveryNode node) { assertNumHandshakes(numHandshakes, serviceB.getOriginalTransport()); assertThat("failed to wait for all nodes to connect", latch.await(5, TimeUnit.SECONDS), equalTo(true)); - serviceA.removeNodeConnectionListener(waitForConnection); - serviceB.removeNodeConnectionListener(waitForConnection); + serviceA.removeConnectionListener(waitForConnection); + serviceB.removeConnectionListener(waitForConnection); } private MockTransportService buildService(final String name, final Version version, ClusterSettings clusterSettings, @@ -633,7 +633,7 @@ public void onNodeDisconnected(DiscoveryNode node) { latch.countDown(); } }; - serviceA.addNodeConnectionListener(disconnectListener); + serviceA.addConnectionListener(disconnectListener); serviceB.close(); assertThat(latch.await(5, TimeUnit.SECONDS), equalTo(true)); } @@ -1715,9 +1715,9 @@ public void onNodeDisconnected(DiscoveryNode node) { fail("disconnect should not be called " + node); } }; - serviceA.addNodeConnectionListener(waitForConnection); - serviceB.addNodeConnectionListener(waitForConnection); - serviceC.addNodeConnectionListener(waitForConnection); + serviceA.addConnectionListener(waitForConnection); + serviceB.addConnectionListener(waitForConnection); + serviceC.addConnectionListener(waitForConnection); serviceC.connectToNode(nodeA); serviceC.connectToNode(nodeB); @@ -1725,9 +1725,9 @@ public void onNodeDisconnected(DiscoveryNode node) { serviceB.connectToNode(nodeC); latch.await(); - serviceA.removeNodeConnectionListener(waitForConnection); - serviceB.removeNodeConnectionListener(waitForConnection); - serviceC.removeNodeConnectionListener(waitForConnection); + serviceA.removeConnectionListener(waitForConnection); + serviceB.removeConnectionListener(waitForConnection); + serviceC.removeConnectionListener(waitForConnection); Map toNodeMap = new HashMap<>(); From aaf2d872c2c389d1f5c8b18e3e02c43155881d2c Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 25 Jul 2018 16:25:21 -0600 Subject: [PATCH 14/29] Use new behaviors --- .../cluster/ClusterInfoServiceIT.java | 32 +++++------ .../discovery/DiscoveryDisruptionIT.java | 24 ++------- .../index/seqno/GlobalCheckpointSyncIT.java | 28 +++------- .../index/store/CorruptedFileIT.java | 52 ++++++++---------- .../index/store/ExceptionRetryIT.java | 14 ++--- .../indices/recovery/IndexRecoveryIT.java | 53 ++++++++----------- .../store/IndicesStoreIntegrationIT.java | 17 +++--- .../elasticsearch/recovery/RelocationIT.java | 13 +++-- .../recovery/TruncatedRecoveryIT.java | 31 +++++------ .../test/transport/MockTransportService.java | 36 +++++++++---- 10 files changed, 122 insertions(+), 178 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index d7d232a08d43c..b17a0cc5418e9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -46,12 +46,9 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import org.hamcrest.Matchers; -import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -116,23 +113,23 @@ public void blockActions(String... actions) { @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() - // manual collection or upon cluster forming. - .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2) - .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), "1s") - .build(); + // manual collection or upon cluster forming. + .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2) + .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), "1s") + .build(); } @Override protected Collection> nodePlugins() { return Arrays.asList(TestPlugin.class, - MockTransportService.TestPlugin.class); + MockTransportService.TestPlugin.class); } public void testClusterInfoServiceCollectsInformation() throws Exception { internalCluster().startNodes(2); assertAcked(prepareCreate("test").setSettings(Settings.builder() - .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0) - .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE).build())); + .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0) + .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE).build())); ensureGreen("test"); InternalTestCluster internalTestCluster = internalCluster(); // Get the cluster info service on the master node @@ -177,8 +174,8 @@ public void testClusterInfoServiceCollectsInformation() throws Exception { public void testClusterInfoServiceInformationClearOnError() throws InterruptedException, ExecutionException { internalCluster().startNodes(2, - // manually control publishing - Settings.builder().put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(), "60m").build()); + // manually control publishing + Settings.builder().put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(), "60m").build()); prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)).get(); ensureGreen("test"); InternalTestCluster internalTestCluster = internalCluster(); @@ -196,19 +193,16 @@ public void testClusterInfoServiceInformationClearOnError() throws InterruptedEx final Set blockedActions = newHashSet(NodesStatsAction.NAME, NodesStatsAction.NAME + "[n]", IndicesStatsAction.NAME, IndicesStatsAction.NAME + "[n]"); // drop all outgoing stats requests to force a timeout. for (DiscoveryNode node : internalTestCluster.clusterService().state().getNodes()) { - mockTransportService.addDelegate(internalTestCluster.getInstance(TransportService.class, node.getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) { - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { + mockTransportService.addSendBehavior(internalTestCluster.getInstance(TransportService.class, node.getName()), + (connection, requestId, action, request, options) -> { if (blockedActions.contains(action)) { if (timeout.get()) { logger.info("dropping [{}] to [{}]", action, node); return; } } - super.sendRequest(connection, requestId, action, request, options); - } - }); + connection.sendRequest(requestId, action, request, options); + }); } // timeouts shouldn't clear the info diff --git a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java index 71ed9100f46c0..65ee9ff075066 100644 --- a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java @@ -37,13 +37,8 @@ import org.elasticsearch.test.disruption.SlowClusterStateProcessing; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.ConnectionProfile; -import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -175,22 +170,11 @@ public void testClusterJoinDespiteOfPublishingIssues() throws Exception { logger.info("allowing requests from non master [{}] to master [{}], waiting for two join request", nonMasterNode, masterNode); final CountDownLatch countDownLatch = new CountDownLatch(2); - nonMasterTransportService.addDelegate(masterTranspotService, new MockTransportService.DelegateTransport(nonMasterTransportService - .original()) { - @Override - protected void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { - if (action.equals(MembershipAction.DISCOVERY_JOIN_ACTION_NAME)) { - countDownLatch.countDown(); - } - super.sendRequest(connection, requestId, action, request, options); + nonMasterTransportService.addSendBehavior(masterTransportService, (connection, requestId, action, request, options) -> { + if (action.equals(MembershipAction.DISCOVERY_JOIN_ACTION_NAME)) { + countDownLatch.countDown(); } - - @Override - public Transport.Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - return super.openConnection(node, profile); - } - + connection.sendRequest(requestId, action, request, options); }); countDownLatch.await(); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java index b2c828cb73f0c..4f4f39c614687 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java @@ -34,11 +34,8 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -85,23 +82,14 @@ public void testBackgroundGlobalCheckpointSync() throws Exception { (MockTransportService) internalCluster().getInstance(TransportService.class, node.getName()); final MockTransportService receiverTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, other.getName()); - - senderTransportService.addDelegate(receiverTransportService, - new MockTransportService.DelegateTransport(senderTransportService.original()) { - @Override - protected void sendRequest( - final Connection connection, - final long requestId, - final String action, - final TransportRequest request, - final TransportRequestOptions options) throws IOException { - if ("indices:admin/seq_no/global_checkpoint_sync[r]".equals(action)) { - throw new IllegalStateException("blocking indices:admin/seq_no/global_checkpoint_sync[r]"); - } else { - super.sendRequest(connection, requestId, action, request, options); - } - } - }); + senderTransportService.addSendBehavior(receiverTransportService, + (connection, requestId, action, request, options) -> { + if ("indices:admin/seq_no/global_checkpoint_sync[r]".equals(action)) { + throw new IllegalStateException("blocking indices:admin/seq_no/global_checkpoint_sync[r]"); + } else { + connection.sendRequest(requestId, action, request, options); + } + }); } } }, diff --git a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java index d3119bbc0fdc4..fdadb5999f77d 100644 --- a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -70,8 +70,6 @@ import org.elasticsearch.test.MockIndexEventListener; import org.elasticsearch.test.store.MockFSIndexStore; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -338,19 +336,15 @@ public void testCorruptionOnNetworkLayerFinalizingRecovery() throws ExecutionExc final CountDownLatch hasCorrupted = new CountDownLatch(1); for (NodeStats dataNode : dataNodeStats) { MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName())); - mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) { - - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException { - if (corrupt.get() && action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { - RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; - byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes; - int i = randomIntBetween(0, req.content().length() - 1); - array[i] = (byte) ~array[i]; // flip one byte in the content - hasCorrupted.countDown(); - } - super.sendRequest(connection, requestId, action, request, options); + mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), (connection, requestId, action, request, options) -> { + if (corrupt.get() && action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { + RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; + byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes; + int i = randomIntBetween(0, req.content().length() - 1); + array[i] = (byte) ~array[i]; // flip one byte in the content + hasCorrupted.countDown(); } + connection.sendRequest(requestId, action, request, options); }); } @@ -410,25 +404,21 @@ public void testCorruptionOnNetworkLayer() throws ExecutionException, Interrupte final boolean truncate = randomBoolean(); for (NodeStats dataNode : dataNodeStats) { MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName())); - mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) { - - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException { - if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { - RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; - if (truncate && req.length() > 1) { - BytesRef bytesRef = req.content().toBytesRef(); - BytesArray array = new BytesArray(bytesRef.bytes, bytesRef.offset, (int) req.length() - 1); - request = new RecoveryFileChunkRequest(req.recoveryId(), req.shardId(), req.metadata(), req.position(), array, req.lastChunk(), req.totalTranslogOps(), req.sourceThrottleTimeInNanos()); - } else { - assert req.content().toBytesRef().bytes == req.content().toBytesRef().bytes : "no internal reference!!"; - final byte[] array = req.content().toBytesRef().bytes; - int i = randomIntBetween(0, req.content().length() - 1); - array[i] = (byte) ~array[i]; // flip one byte in the content - } + mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), (connection, requestId, action, request, options) -> { + if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { + RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; + if (truncate && req.length() > 1) { + BytesRef bytesRef = req.content().toBytesRef(); + BytesArray array = new BytesArray(bytesRef.bytes, bytesRef.offset, (int) req.length() - 1); + request = new RecoveryFileChunkRequest(req.recoveryId(), req.shardId(), req.metadata(), req.position(), array, req.lastChunk(), req.totalTranslogOps(), req.sourceThrottleTimeInNanos()); + } else { + assert req.content().toBytesRef().bytes == req.content().toBytesRef().bytes : "no internal reference!!"; + final byte[] array = req.content().toBytesRef().bytes; + int i = randomIntBetween(0, req.content().length() - 1); + array[i] = (byte) ~array[i]; // flip one byte in the content } - super.sendRequest(connection, requestId, action, request, options); } + connection.sendRequest(requestId, action, request, options); }); } diff --git a/server/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/server/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java index 86519acf0926a..017fb360de58a 100644 --- a/server/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java +++ b/server/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java @@ -38,8 +38,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.ConnectTransportException; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -95,18 +93,14 @@ public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, I for (NodeStats dataNode : nodeStats.getNodes()) { MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName())); - mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), - new MockTransportService.DelegateTransport(mockTransportService.original()) { - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { - super.sendRequest(connection, requestId, action, request, options); + mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), + (connection, requestId, action, request, options) -> { + connection.sendRequest(requestId, action, request, options); if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) { logger.debug("Throw ConnectTransportException"); throw new ConnectTransportException(connection.getNode(), action); } - } - }); + }); } BulkRequestBuilder bulkBuilder = client.prepareBulk(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index cf1449fecd6a5..7e7f2981b4586 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -598,8 +598,8 @@ public void testDisconnectsWhileRecovering() throws Exception { TransportService blueTransportService = internalCluster().getInstance(TransportService.class, blueNodeName); final CountDownLatch requestBlocked = new CountDownLatch(1); - blueMockTransportService.addDelegate(redTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, blueMockTransportService.original(), requestBlocked)); - redMockTransportService.addDelegate(blueTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, redMockTransportService.original(), requestBlocked)); + blueMockTransportService.addSendBehavior(redTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, requestBlocked)); + redMockTransportService.addSendBehavior(blueTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, requestBlocked)); logger.info("--> starting recovery from blue to red"); client().admin().indices().prepareUpdateSettings(indexName).setSettings( @@ -620,21 +620,20 @@ public void testDisconnectsWhileRecovering() throws Exception { } - private class RecoveryActionBlocker extends MockTransportService.DelegateTransport { + private class RecoveryActionBlocker implements MockTransportService.SendRequestBehavior { private final boolean dropRequests; private final String recoveryActionToBlock; private final CountDownLatch requestBlocked; - RecoveryActionBlocker(boolean dropRequests, String recoveryActionToBlock, Transport delegate, CountDownLatch requestBlocked) { - super(delegate); + RecoveryActionBlocker(boolean dropRequests, String recoveryActionToBlock, CountDownLatch requestBlocked) { this.dropRequests = dropRequests; this.recoveryActionToBlock = recoveryActionToBlock; this.requestBlocked = requestBlocked; } @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { + public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, + TransportRequestOptions options) throws IOException { if (recoveryActionToBlock.equals(action) || requestBlocked.getCount() == 0) { logger.info("--> preventing {} request", action); requestBlocked.countDown(); @@ -643,7 +642,7 @@ protected void sendRequest(Connection connection, long requestId, String action, } throw new ConnectTransportException(connection.getNode(), "DISCONNECT: prevented " + action + " request"); } - super.sendRequest(connection, requestId, action, request, options); + connection.sendRequest(requestId, action, request, options); } } @@ -686,12 +685,12 @@ public void testDisconnectsDuringRecovery() throws Exception { MockTransportService blueMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, blueNodeName); MockTransportService redMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, redNodeName); - redMockTransportService.addDelegate(blueMockTransportService, new MockTransportService.DelegateTransport(redMockTransportService.original()) { + redMockTransportService.addSendBehavior(blueMockTransportService, new MockTransportService.SendRequestBehavior() { private final AtomicInteger count = new AtomicInteger(); @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { + public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, + TransportRequestOptions options) throws IOException { logger.info("--> sending request {} on {}", action, connection.getNode()); if (PeerRecoverySourceService.Actions.START_RECOVERY.equals(action) && count.incrementAndGet() == 1) { // ensures that it's considered as valid recovery attempt by source @@ -701,7 +700,7 @@ protected void sendRequest(Connection connection, long requestId, String action, } catch (InterruptedException e) { throw new RuntimeException(e); } - super.sendRequest(connection, requestId, action, request, options); + connection.sendRequest(requestId, action, request, options); try { Thread.sleep(disconnectAfterDelay.millis()); } catch (InterruptedException e) { @@ -709,35 +708,27 @@ protected void sendRequest(Connection connection, long requestId, String action, } throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulation disconnect after successfully sending " + action + " request"); } else { - super.sendRequest(connection, requestId, action, request, options); + connection.sendRequest(requestId, action, request, options); } } }); final AtomicBoolean finalized = new AtomicBoolean(); - blueMockTransportService.addDelegate(redMockTransportService, new MockTransportService.DelegateTransport(blueMockTransportService.original()) { - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { - logger.info("--> sending request {} on {}", action, connection.getNode()); - if (action.equals(PeerRecoveryTargetService.Actions.FINALIZE)) { - finalized.set(true); - } - super.sendRequest(connection, requestId, action, request, options); + blueMockTransportService.addSendBehavior(redMockTransportService, (connection, requestId, action, request, options) -> { + logger.info("--> sending request {} on {}", action, connection.getNode()); + if (action.equals(PeerRecoveryTargetService.Actions.FINALIZE)) { + finalized.set(true); } + connection.sendRequest(requestId, action, request, options); }); for (MockTransportService mockTransportService : Arrays.asList(redMockTransportService, blueMockTransportService)) { - mockTransportService.addDelegate(masterTransportService, new MockTransportService.DelegateTransport(mockTransportService.original()) { - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { - logger.info("--> sending request {} on {}", action, connection.getNode()); - if ((primaryRelocation && finalized.get()) == false) { - assertNotEquals(action, ShardStateAction.SHARD_FAILED_ACTION_NAME); - } - super.sendRequest(connection, requestId, action, request, options); + mockTransportService.addSendBehavior(masterTransportService, (connection, requestId, action, request, options) -> { + logger.info("--> sending request {} on {}", action, connection.getNode()); + if ((primaryRelocation && finalized.get()) == false) { + assertNotEquals(action, ShardStateAction.SHARD_FAILED_ACTION_NAME); } + connection.sendRequest(requestId, action, request, options); }); } diff --git a/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java index 5c1c0d91e608b..7e99ccbbe6117 100644 --- a/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java @@ -55,8 +55,6 @@ import org.elasticsearch.test.disruption.BlockClusterStateProcessing; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.ConnectTransportException; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -234,16 +232,13 @@ public void testShardCleanupIfShardDeletionAfterRelocationFailedAndIndexDeleted( MockTransportService transportServiceNode_1 = (MockTransportService) internalCluster().getInstance(TransportService.class, node_1); TransportService transportServiceNode_2 = internalCluster().getInstance(TransportService.class, node_2); final CountDownLatch shardActiveRequestSent = new CountDownLatch(1); - transportServiceNode_1.addDelegate(transportServiceNode_2, new MockTransportService.DelegateTransport(transportServiceNode_1.original()) { - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException { - if (action.equals("internal:index/shard/exists") && shardActiveRequestSent.getCount() > 0) { - shardActiveRequestSent.countDown(); - logger.info("prevent shard active request from being sent"); - throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulated"); - } - super.sendRequest(connection, requestId, action, request, options); + transportServiceNode_1.addSendBehavior(transportServiceNode_2, (connection, requestId, action, request, options) -> { + if (action.equals("internal:index/shard/exists") && shardActiveRequestSent.getCount() > 0) { + shardActiveRequestSent.countDown(); + logger.info("prevent shard active request from being sent"); + throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulated"); } + connection.sendRequest(requestId, action, request, options); }); logger.info("--> move shard from {} to {}, and wait for relocation to finish", node_1, node_2); diff --git a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 738cd00a43d2e..6998bc81a4105 100644 --- a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -372,7 +372,7 @@ public void testCancellationCleansTempFiles() throws Exception { MockTransportService mockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, p_node); for (DiscoveryNode node : clusterService.state().nodes()) { if (!node.equals(clusterService.localNode())) { - mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, node.getName()), new RecoveryCorruption(mockTransportService.original(), corruptionCount)); + mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, node.getName()), new RecoveryCorruption(corruptionCount)); } } @@ -485,17 +485,16 @@ public void testIndexAndRelocateConcurrently() throws ExecutionException, Interr } - class RecoveryCorruption extends MockTransportService.DelegateTransport { + class RecoveryCorruption implements MockTransportService.SendRequestBehavior { private final CountDownLatch corruptionCount; - RecoveryCorruption(Transport transport, CountDownLatch corruptionCount) { - super(transport); + RecoveryCorruption(CountDownLatch corruptionCount) { this.corruptionCount = corruptionCount; } @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException { + public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException { if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { RecoveryFileChunkRequest chunkRequest = (RecoveryFileChunkRequest) request; if (chunkRequest.name().startsWith(IndexFileNames.SEGMENTS)) { @@ -506,9 +505,9 @@ protected void sendRequest(Connection connection, long requestId, String action, array[0] = (byte) ~array[0]; // flip one byte in the content corruptionCount.countDown(); } - super.sendRequest(connection, requestId, action, request, options); + connection.sendRequest(requestId, action, request, options); } else { - super.sendRequest(connection, requestId, action, request, options); + connection.sendRequest(requestId, action, request, options); } } } diff --git a/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java index b33cfb508b930..ac8688c9847d3 100644 --- a/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java @@ -35,11 +35,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -70,8 +67,8 @@ protected Collection> nodePlugins() { */ public void testCancelRecoveryAndResume() throws Exception { assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder() - .put(CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES))) - .get().isAcknowledged()); + .put(CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES))) + .get().isAcknowledged()); NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get(); List dataNodeStats = new ArrayList<>(); @@ -91,9 +88,9 @@ public void testCancelRecoveryAndResume() throws Exception { // create the index and prevent allocation on any other nodes than the lucky one // we have no replicas so far and make sure that we allocate the primary on the lucky node assertAcked(prepareCreate("test") - .addMapping("type1", "field1", "type=text", "the_id", "type=text") - .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards()) - .put("index.routing.allocation.include._name", primariesNode.getNode().getName()))); // only allocate on the lucky node + .addMapping("type1", "field1", "type=text", "the_id", "type=text") + .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards()) + .put("index.routing.allocation.include._name", primariesNode.getNode().getName()))); // only allocate on the lucky node // index some docs and check if they are coming back int numDocs = randomIntBetween(100, 200); @@ -116,11 +113,8 @@ public void testCancelRecoveryAndResume() throws Exception { final AtomicBoolean truncate = new AtomicBoolean(true); for (NodeStats dataNode : dataNodeStats) { MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName())); - mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) { - - @Override - protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { + mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), + (connection, requestId, action, request, options) -> { if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; logger.debug("file chunk [{}] lastChunk: {}", req, req.lastChunk()); @@ -129,16 +123,15 @@ protected void sendRequest(Connection connection, long requestId, String action, throw new RuntimeException("Caused some truncated files for fun and profit"); } } - super.sendRequest(connection, requestId, action, request, options); - } - }); + connection.sendRequest(requestId, action, request, options); + }); } logger.info("--> bumping replicas to 1"); // client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put("index.routing.allocation.include._name", // now allow allocation on all nodes - primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName())).get(); + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.routing.allocation.include._name", // now allow allocation on all nodes + primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName())).get(); latch.await(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index f2235b0adc32d..178d033586b8e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -47,7 +47,6 @@ import org.elasticsearch.test.tasks.MockTaskManager; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectTransportException; -import org.elasticsearch.transport.ConnectionManager; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.MockTcpTransport; import org.elasticsearch.transport.RequestHandlerRegistry; @@ -405,7 +404,7 @@ public void clearCallback() { /** * Adds a new delegate transport that is used for communication with the given transport service. * - * @return {@code true} iff no other delegate was registered for any of the addresses bound by transport service. + * @return {@code true} if no other send behavior was registered for any of the addresses bound by transport service. */ public boolean addDelegate(TransportService transportService, DelegateTransport transport) { boolean noRegistered = true; @@ -415,15 +414,37 @@ public boolean addDelegate(TransportService transportService, DelegateTransport return noRegistered; } + /** + * Adds a new send behavior that is used for communication with the given transport service. + * + * @return {@code true} if no other delegate was registered for any of the addresses bound by transport service. + */ + public boolean addSendBehavior(TransportService transportService, SendRequestBehavior sendBehavior) { + boolean noRegistered = true; + for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { + noRegistered &= addSendBehavior(transportAddress, sendBehavior); + } + return noRegistered; + } + /** * Adds a new delegate transport that is used for communication with the given transport address. * - * @return {@code true} iff no other delegate was registered for this address before. + * @return {@code true} if no other delegate was registered for this address before. */ public boolean addDelegate(TransportAddress transportAddress, DelegateTransport transport) { return transport().transports.put(transportAddress, transport) == null; } + /** + * Adds a new send behavior that is used for communication with the given transport address. + * + * @return {@code true} if no other send behavior was registered for this address before. + */ + public boolean addSendBehavior(TransportAddress transportAddress, SendRequestBehavior sendBehavior) { + return transport().addSendBehavior(transportAddress, sendBehavior); + } + private LookupTestTransport transport() { return (LookupTestTransport) transport; } @@ -448,8 +469,8 @@ public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) return new WrappedConnection(transportToUse.openConnection(node, profile)); } - void addSendBehavior(TransportAddress transportAddress, MockTransportService.SendRequestBehavior sendBehavior) { - sendBehaviors.put(transportAddress, sendBehavior); + boolean addSendBehavior(TransportAddress transportAddress, MockTransportService.SendRequestBehavior sendBehavior) { + return sendBehaviors.put(transportAddress, sendBehavior) == null; } void clearSendBehaviors() { @@ -629,11 +650,6 @@ public void close() { public Map profileBoundAddresses() { return transport.profileBoundAddresses(); } - - protected void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { - connection.sendRequest(requestId, action, request, options); - } } List activeTracers = new CopyOnWriteArrayList<>(); From f5778c0ac4eb5d5db6d492adc0742295c5a0864d Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 25 Jul 2018 16:46:29 -0600 Subject: [PATCH 15/29] Fix some tests --- .../org/elasticsearch/test/transport/MockTransportService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 178d033586b8e..3a6565b388dcc 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -528,7 +528,7 @@ public boolean isClosed() { @Override public Version getVersion() { - return getNode().getVersion(); + return connection.getVersion(); } @Override From 95a7b856bcaeea555ef33ae05f195a5bda0e4fa9 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 26 Jul 2018 17:39:17 -0600 Subject: [PATCH 16/29] Working on fixing tests --- .../transport/TransportService.java | 2 +- .../indices/get/GetIndexActionTests.java | 22 +- .../TransportClientNodesServiceTests.java | 23 +- .../RemoteClusterConnectionTests.java | 182 ++++++----- .../test/transport/MockTransportService.java | 283 ++++++++++++++---- .../test/transport/NewCapturingTransport.java | 185 ++++++++++++ 6 files changed, 522 insertions(+), 175 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/transport/NewCapturingTransport.java diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 5ddd61b5040a3..48c743604e8e4 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -79,6 +79,7 @@ public class TransportService extends AbstractLifecycleComponent implements Tran private final CountDownLatch blockIncomingRequestsLatch = new CountDownLatch(1); protected final Transport transport; + protected final ConnectionManager connectionManager; protected final ThreadPool threadPool; protected final ClusterName clusterName; protected final TaskManager taskManager; @@ -87,7 +88,6 @@ public class TransportService extends AbstractLifecycleComponent implements Tran private final boolean connectToRemoteCluster; private final Transport.ResponseHandlers responseHandlers; private final TransportInterceptor interceptor; - private final ConnectionManager connectionManager; // An LRU (don't really care about concurrency here) that holds the latest timed out requests so if they // do show up, we can print more descriptive information about them diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java index 02a98eacda0e9..d8b9a275be13d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java @@ -36,6 +36,8 @@ import org.elasticsearch.test.transport.CapturingTransport; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import org.junit.After; import org.junit.Before; @@ -81,12 +83,10 @@ public void tearDown() throws Exception { public void testIncludeDefaults() { GetIndexRequest defaultsRequest = new GetIndexRequest().indices(indexName).includeDefaults(true); getIndexAction.execute(null, defaultsRequest, ActionListener.wrap( - defaultsResponse -> { - assertNotNull( - "index.refresh_interval should be set as we are including defaults", - defaultsResponse.getSetting(indexName, "index.refresh_interval") - ); - }, exception -> { + defaultsResponse -> assertNotNull( + "index.refresh_interval should be set as we are including defaults", + defaultsResponse.getSetting(indexName, "index.refresh_interval") + ), exception -> { throw new AssertionError(exception); }) ); @@ -95,12 +95,10 @@ public void testIncludeDefaults() { public void testDoNotIncludeDefaults() { GetIndexRequest noDefaultsRequest = new GetIndexRequest().indices(indexName); getIndexAction.execute(null, noDefaultsRequest, ActionListener.wrap( - noDefaultsResponse -> { - assertNull( - "index.refresh_interval should be null as it was never set", - noDefaultsResponse.getSetting(indexName, "index.refresh_interval") - ); - }, exception -> { + noDefaultsResponse -> assertNull( + "index.refresh_interval should be null as it was never set", + noDefaultsResponse.getSetting(indexName, "index.refresh_interval") + ), exception -> { throw new AssertionError(exception); }) ); diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index f06f3a8a66d1f..038296793295d 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ConnectionManager; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportChannel; @@ -359,20 +360,16 @@ public void testSniffNodesSamplerClosesConnections() throws Exception { final List establishedConnections = new CopyOnWriteArrayList<>(); final List reusedConnections = new CopyOnWriteArrayList<>(); - clientService.addDelegate(remoteService, new MockTransportService.DelegateTransport(clientService.original()) { - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - MockConnection connection = new MockConnection(super.openConnection(node, profile)); - establishedConnections.add(connection); - return connection; - } + clientService.addGetConnectionBehavior(remoteService, (connectionManager, discoveryNode) -> { + MockConnection connection = new MockConnection(connectionManager.getConnection(discoveryNode)); + reusedConnections.add(connection); + return connection; + }); -// @Override -// public Connection getConnection(DiscoveryNode node) { -// MockConnection connection = new MockConnection(super.getConnection(node)); -// reusedConnections.add(connection); -// return connection; -// } + clientService.addConnectBehavior(remoteService, (transport, discoveryNode, profile) -> { + MockConnection connection = new MockConnection(transport.openConnection(discoveryNode, profile)); + establishedConnections.add(connection); + return connection; }); clientService.start(); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 15d1817f832af..4de4314979ca2 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -105,34 +105,34 @@ public static MockTransportService startTransport(String id, List } public static MockTransportService startTransport( - final String id, - final List knownNodes, - final Version version, - final ThreadPool threadPool, - final Settings settings) { + final String id, + final List knownNodes, + final Version version, + final ThreadPool threadPool, + final Settings settings) { boolean success = false; final Settings s = Settings.builder().put(settings).put("node.name", id).build(); ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(s); MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, null); try { - newService.registerRequestHandler(ClusterSearchShardsAction.NAME,ThreadPool.Names.SAME, ClusterSearchShardsRequest::new, + newService.registerRequestHandler(ClusterSearchShardsAction.NAME, ThreadPool.Names.SAME, ClusterSearchShardsRequest::new, (request, channel, task) -> { - if ("index_not_found".equals(request.preference())) { - channel.sendResponse(new IndexNotFoundException("index")); - } else { - channel.sendResponse(new ClusterSearchShardsResponse(new ClusterSearchShardsGroup[0], - knownNodes.toArray(new DiscoveryNode[0]), Collections.emptyMap())); - } - }); + if ("index_not_found".equals(request.preference())) { + channel.sendResponse(new IndexNotFoundException("index")); + } else { + channel.sendResponse(new ClusterSearchShardsResponse(new ClusterSearchShardsGroup[0], + knownNodes.toArray(new DiscoveryNode[0]), Collections.emptyMap())); + } + }); newService.registerRequestHandler(ClusterStateAction.NAME, ThreadPool.Names.SAME, ClusterStateRequest::new, (request, channel, task) -> { - DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); - for (DiscoveryNode node : knownNodes) { - builder.add(node); - } - ClusterState build = ClusterState.builder(clusterName).nodes(builder.build()).build(); - channel.sendResponse(new ClusterStateResponse(clusterName, build, 0L)); - }); + DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); + for (DiscoveryNode node : knownNodes) { + builder.add(node); + } + ClusterState build = ClusterState.builder(clusterName).nodes(builder.build()).build(); + channel.sendResponse(new ClusterStateResponse(clusterName, build, 0L)); + }); newService.start(); newService.acceptIncomingRequests(); success = true; @@ -440,15 +440,14 @@ public void close() throws IOException { // no-op } }; - service.addDelegate(seedNode.getAddress(), new MockTransportService.DelegateTransport(service.original()) { -// @Override -// public Connection getConnection(DiscoveryNode node) { -// if (node == seedNode) { -// return seedConnection; -// } -// return super.getConnection(node); -// } + + service.addGetConnectionBehavior(seedNode.getAddress(), (connectionManager, discoveryNode) -> { + if (discoveryNode == seedNode) { + return seedConnection; + } + return connectionManager.getConnection(discoveryNode); }); + service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", @@ -511,7 +510,9 @@ public void run() { closeRemote.countDown(); listenerCalled.await(); assertNotNull(exceptionReference.get()); - expectThrows(CancellableThreads.ExecutionCancelledException.class, () -> {throw exceptionReference.get();}); + expectThrows(CancellableThreads.ExecutionCancelledException.class, () -> { + throw exceptionReference.get(); + }); } } @@ -545,7 +546,7 @@ public void testFetchShards() throws Exception { .indicesOptions(request.indicesOptions()).local(true).preference(request.preference()) .routing(request.routing()); connection.fetchSearchShards(searchShardsRequest, - new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch)); + new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch)); responseLatch.await(); assertNull(failReference.get()); assertNotNull(reference.get()); @@ -624,18 +625,18 @@ public void testFetchShardsSkipUnavailable() throws Exception { service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Collections.singletonList(seedNode), service, Integer.MAX_VALUE, n -> true)) { + Collections.singletonList(seedNode), service, Integer.MAX_VALUE, n -> true)) { SearchRequest request = new SearchRequest("test-index"); ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest("test-index") - .indicesOptions(request.indicesOptions()).local(true).preference(request.preference()) - .routing(request.routing()); + .indicesOptions(request.indicesOptions()).local(true).preference(request.preference()) + .routing(request.routing()); { CountDownLatch responseLatch = new CountDownLatch(1); AtomicReference reference = new AtomicReference<>(); AtomicReference failReference = new AtomicReference<>(); connection.fetchSearchShards(searchShardsRequest, - new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch)); + new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch)); assertTrue(responseLatch.await(5, TimeUnit.SECONDS)); assertNull(failReference.get()); assertNotNull(reference.get()); @@ -664,9 +665,9 @@ public void onNodeDisconnected(DiscoveryNode node) { AtomicReference reference = new AtomicReference<>(); AtomicReference failReference = new AtomicReference<>(); connection.fetchSearchShards(searchShardsRequest, - new LatchedActionListener<>(ActionListener.wrap((s) -> { - reference.set(s); - }, failReference::set), responseLatch)); + new LatchedActionListener<>(ActionListener.wrap((s) -> { + reference.set(s); + }, failReference::set), responseLatch)); assertTrue(responseLatch.await(1, TimeUnit.SECONDS)); assertNotNull(failReference.get()); assertNull(reference.get()); @@ -679,7 +680,7 @@ public void onNodeDisconnected(DiscoveryNode node) { AtomicReference reference = new AtomicReference<>(); AtomicReference failReference = new AtomicReference<>(); connection.fetchSearchShards(searchShardsRequest, - new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch)); + new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch)); assertTrue(responseLatch.await(1, TimeUnit.SECONDS)); assertNull(failReference.get()); assertNotNull(reference.get()); @@ -702,7 +703,7 @@ public void onNodeDisconnected(DiscoveryNode node) { AtomicReference reference = new AtomicReference<>(); AtomicReference failReference = new AtomicReference<>(); connection.fetchSearchShards(searchShardsRequest, - new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch)); + new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch)); assertTrue(responseLatch.await(1, TimeUnit.SECONDS)); assertNull(failReference.get()); assertNotNull(reference.get()); @@ -749,27 +750,28 @@ public void run() { for (int i = 0; i < numConnectionAttempts; i++) { AtomicBoolean executed = new AtomicBoolean(false); ActionListener listener = ActionListener.wrap( - x -> { - assertTrue(executed.compareAndSet(false, true)); - latch.countDown();}, - x -> { - /* - * This can occur on a thread submitted to the thread pool while we are closing the - * remote cluster connection at the end of the test. - */ - if (x instanceof CancellableThreads.ExecutionCancelledException) { - // we should already be shutting down - assertTrue(executed.get()); - return; - } + x -> { + assertTrue(executed.compareAndSet(false, true)); + latch.countDown(); + }, + x -> { + /* + * This can occur on a thread submitted to the thread pool while we are closing the + * remote cluster connection at the end of the test. + */ + if (x instanceof CancellableThreads.ExecutionCancelledException) { + // we should already be shutting down + assertTrue(executed.get()); + return; + } - assertTrue(executed.compareAndSet(false, true)); - latch.countDown(); + assertTrue(executed.compareAndSet(false, true)); + latch.countDown(); - if (!(x instanceof RejectedExecutionException)) { - throw new AssertionError(x); - } - }); + if (!(x instanceof RejectedExecutionException)) { + throw new AssertionError(x); + } + }); connection.updateSeedNodes(seedNodes, listener); } latch.await(); @@ -856,7 +858,7 @@ public void run() { } else { throw new AssertionError(x); } - }); + }); try { connection.updateSeedNodes(seedNodes, listener); } catch (Exception e) { @@ -900,7 +902,7 @@ public void testGetConnectionInfo() throws Exception { service.acceptIncomingRequests(); int maxNumConnections = randomIntBetween(1, 5); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes, service, maxNumConnections, n -> true)) { + seedNodes, service, maxNumConnections, n -> true)) { // test no nodes connected RemoteConnectionInfo remoteConnectionInfo = assertSerialization(connection.getConnectionInfo()); assertNotNull(remoteConnectionInfo); @@ -975,16 +977,16 @@ private static RemoteConnectionInfo assertSerialization(RemoteConnectionInfo inf RemoteConnectionInfo remoteConnectionInfo = new RemoteConnectionInfo(in); assertEquals(info, remoteConnectionInfo); assertEquals(info.hashCode(), remoteConnectionInfo.hashCode()); - return randomBoolean() ? info : remoteConnectionInfo; + return randomBoolean() ? info : remoteConnectionInfo; } } public void testRemoteConnectionInfoBwComp() throws IOException { final Version version = VersionUtils.randomVersionBetween(random(), - Version.V_6_1_0, VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1)); + Version.V_6_1_0, VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1)); RemoteConnectionInfo expected = new RemoteConnectionInfo("test_cluster", - Collections.singletonList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - 4, 4, new TimeValue(30, TimeUnit.MINUTES), false); + Collections.singletonList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), + 4, 4, new TimeValue(30, TimeUnit.MINUTES), false); // This version was created using the serialization code in use from 6.1 but before 7.0 String encoded = "AQQAAAAABzAuMC4wLjAAAAABAQQAAAAABzAuMC4wLjAAAABQBDwEBAx0ZXN0X2NsdXN0ZXIA"; @@ -1009,7 +1011,7 @@ public void testRemoteConnectionInfoBwComp() throws IOException { public void testRenderConnectionInfoXContent() throws IOException { RemoteConnectionInfo stats = new RemoteConnectionInfo("test_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS,1)), + Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), 4, 3, TimeValue.timeValueMinutes(30), true); stats = assertSerialization(stats); XContentBuilder builder = XContentFactory.jsonBuilder(); @@ -1018,10 +1020,10 @@ public void testRenderConnectionInfoXContent() throws IOException { builder.endObject(); assertEquals("{\"test_cluster\":{\"seeds\":[\"0.0.0.0:1\"],\"connected\":true," + "\"num_nodes_connected\":3,\"max_connections_per_cluster\":4,\"initial_connect_timeout\":\"30m\"," + - "\"skip_unavailable\":true}}", Strings.toString(builder)); + "\"skip_unavailable\":true}}", Strings.toString(builder)); stats = new RemoteConnectionInfo("some_other_cluster", - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS,1), new TransportAddress(TransportAddress.META_ADDRESS,2)), + Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1), new TransportAddress(TransportAddress.META_ADDRESS, 2)), 2, 0, TimeValue.timeValueSeconds(30), false); stats = assertSerialization(stats); builder = XContentFactory.jsonBuilder(); @@ -1029,14 +1031,14 @@ public void testRenderConnectionInfoXContent() throws IOException { stats.toXContent(builder, null); builder.endObject(); assertEquals("{\"some_other_cluster\":{\"seeds\":[\"0.0.0.0:1\",\"0.0.0.0:2\"]," - + "\"connected\":false,\"num_nodes_connected\":0,\"max_connections_per_cluster\":2,\"initial_connect_timeout\":\"30s\"," + - "\"skip_unavailable\":false}}", Strings.toString(builder)); + + "\"connected\":false,\"num_nodes_connected\":0,\"max_connections_per_cluster\":2,\"initial_connect_timeout\":\"30s\"," + + "\"skip_unavailable\":false}}", Strings.toString(builder)); } public void testEnsureConnected() throws IOException, InterruptedException { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { + MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); knownNodes.add(seedTransport.getLocalDiscoNode()); @@ -1131,7 +1133,7 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted try { final int numDiscoverableNodes = randomIntBetween(5, 20); List discoverableNodes = new ArrayList<>(numDiscoverableNodes); - for (int i = 0; i < numDiscoverableNodes; i++ ) { + for (int i = 0; i < numDiscoverableNodes; i++) { MockTransportService transportService = startTransport("discoverable_node" + i, knownNodes, Version.CURRENT); discoverableNodes.add(transportService.getLocalDiscoNode()); discoverableTransports.add(transportService); @@ -1216,12 +1218,12 @@ public void testClusterNameIsChecked() throws Exception { Settings settings = Settings.builder().put("cluster.name", "testClusterNameIsChecked").build(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT, threadPool, settings); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, threadPool, - settings); - MockTransportService otherClusterTransport = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, - Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build()); - MockTransportService otherClusterDiscoverable= startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, - Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build())) { + MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, threadPool, + settings); + MockTransportService otherClusterTransport = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, + Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build()); + MockTransportService otherClusterDiscoverable = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, + Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build())) { DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); knownNodes.add(seedTransport.getLocalDiscoNode()); @@ -1281,7 +1283,7 @@ public DiscoveryNode getNode() { @Override public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws TransportException { + throws TransportException { // no-op } @@ -1290,24 +1292,20 @@ public void close() { // no-op } }; - service.addDelegate(connectedNode.getAddress(), new MockTransportService.DelegateTransport(service.original()) { -// @Override -// public Connection getConnection(DiscoveryNode node) { -// if (node == connectedNode) { -// return seedConnection; -// } -// return super.getConnection(node); -// } -// -// @Override -// public boolean nodeConnected(DiscoveryNode node) { -// return node.equals(connectedNode); -// } + + service.addNodeConnectedBehavior(connectedNode.getAddress(), (connectionManager, discoveryNode) + -> discoveryNode.equals(connectedNode)); + + service.addGetConnectionBehavior(connectedNode.getAddress(), (connectionManager, discoveryNode) -> { + if (discoveryNode == connectedNode) { + return seedConnection; + } + return connectionManager.getConnection(discoveryNode); }); service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Collections.singletonList(connectedNode), service, Integer.MAX_VALUE, n -> true)) { + Collections.singletonList(connectedNode), service, Integer.MAX_VALUE, n -> true)) { connection.addConnectedNode(connectedNode); for (int i = 0; i < 10; i++) { //always a direct connection as the remote node is already connected diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 3a6565b388dcc..c3c7a348b903b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.component.Lifecycle; @@ -47,6 +48,7 @@ import org.elasticsearch.test.tasks.MockTaskManager; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.ConnectionManager; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.MockTcpTransport; import org.elasticsearch.transport.RequestHandlerRegistry; @@ -148,8 +150,15 @@ public MockTransportService(Settings settings, Transport transport, ThreadPool t public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, Set taskHeaders) { - super(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); - this.original = transport; + this(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); + } + + private MockTransportService(Settings settings, LookupTestTransport transport, ThreadPool threadPool, TransportInterceptor interceptor, + Function localNodeFactory, + @Nullable ClusterSettings clusterSettings, Set taskHeaders) { + super(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders, + new MockConnectionManager(new ConnectionManager(settings, transport, threadPool), settings, transport, threadPool)); + this.original = transport.transport; } public static TransportAddress[] extractTransportAddresses(TransportService transportService) { @@ -173,7 +182,8 @@ protected TaskManager createTaskManager(Settings settings, ThreadPool threadPool * Clears all the registered rules. */ public void clearAllRules() { - transport().clearSendBehaviors(); + transport().clearBehaviors(); + connectionManager().clearBehaviors(); transport().transports.clear(); } @@ -190,8 +200,9 @@ public void clearRule(TransportService transportService) { * Clears the rule associated with the provided transport address. */ public void clearRule(TransportAddress transportAddress) { + transport().clearBehavior(transportAddress); + connectionManager().clearBehavior(transportAddress); transport().transports.remove(transportAddress); - transport().clearSendBehavior(transportAddress); } /** @@ -216,12 +227,8 @@ public void addFailToSendNoConnectRule(TransportService transportService) { * is added to fail as well. */ public void addFailToSendNoConnectRule(TransportAddress transportAddress) { - addDelegate(transportAddress, new DelegateTransport(original) { - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - throw new ConnectTransportException(node, "DISCONNECT: simulated"); - } + transport().addConnectBehavior(transportAddress, (transport, discoveryNode, profile) -> { + throw new ConnectTransportException(discoveryNode, "DISCONNECT: simulated"); }); transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { @@ -282,12 +289,8 @@ public void addUnresponsiveRule(TransportService transportService) { * and failing to connect once the rule was added. */ public void addUnresponsiveRule(TransportAddress transportAddress) { - addDelegate(transportAddress, new DelegateTransport(original) { - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); - } + transport().addConnectBehavior(transportAddress, (transport, discoveryNode, profile) -> { + throw new ConnectTransportException(discoveryNode, "UNRESPONSIVE: simulated"); }); transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> { @@ -318,28 +321,24 @@ public void addUnresponsiveRule(TransportAddress transportAddress, final TimeVal Supplier delaySupplier = () -> new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime)); - addDelegate(transportAddress, new DelegateTransport(original) { - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - TimeValue delay = delaySupplier.get(); - if (delay.millis() <= 0) { - return original.openConnection(node, profile); - } + transport().addConnectBehavior(transportAddress, (transport, discoveryNode, profile) -> { + TimeValue delay = delaySupplier.get(); + if (delay.millis() <= 0) { + return original.openConnection(discoveryNode, profile); + } - // TODO: Replace with proper setting - TimeValue connectingTimeout = TcpTransport.TCP_CONNECT_TIMEOUT.getDefault(Settings.EMPTY); - try { - if (delay.millis() < connectingTimeout.millis()) { - Thread.sleep(delay.millis()); - return original.openConnection(node, profile); - } else { - Thread.sleep(connectingTimeout.millis()); - throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); - } - } catch (InterruptedException e) { - throw new ConnectTransportException(node, "UNRESPONSIVE: simulated"); + // TODO: Replace with proper setting + TimeValue connectingTimeout = TcpTransport.TCP_CONNECT_TIMEOUT.getDefault(Settings.EMPTY); + try { + if (delay.millis() < connectingTimeout.millis()) { + Thread.sleep(delay.millis()); + return original.openConnection(discoveryNode, profile); + } else { + Thread.sleep(connectingTimeout.millis()); + throw new ConnectTransportException(discoveryNode, "UNRESPONSIVE: simulated"); } + } catch (InterruptedException e) { + throw new ConnectTransportException(discoveryNode, "UNRESPONSIVE: simulated"); } }); @@ -402,53 +401,192 @@ public void clearCallback() { } /** - * Adds a new delegate transport that is used for communication with the given transport service. + * Adds a new send behavior that is used for communication with the given transport service. * * @return {@code true} if no other send behavior was registered for any of the addresses bound by transport service. */ - public boolean addDelegate(TransportService transportService, DelegateTransport transport) { + public boolean addSendBehavior(TransportService transportService, SendRequestBehavior sendBehavior) { boolean noRegistered = true; for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { - noRegistered &= addDelegate(transportAddress, transport); + noRegistered &= addSendBehavior(transportAddress, sendBehavior); } return noRegistered; } /** - * Adds a new send behavior that is used for communication with the given transport service. + * Adds a new send behavior that is used for communication with the given transport address. * - * @return {@code true} if no other delegate was registered for any of the addresses bound by transport service. + * @return {@code true} if no other send behavior was registered for this address before. */ - public boolean addSendBehavior(TransportService transportService, SendRequestBehavior sendBehavior) { + public boolean addSendBehavior(TransportAddress transportAddress, SendRequestBehavior sendBehavior) { + return transport().addSendBehavior(transportAddress, sendBehavior); + } + + + /** + * Adds a new connect behavior that is used for creating connections with the given transport service. + * + * @return {@code true} if no other send behavior was registered for any of the addresses bound by transport service. + */ + public boolean addConnectBehavior(TransportService transportService, ConnectBehavior connectBehavior) { boolean noRegistered = true; for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { - noRegistered &= addSendBehavior(transportAddress, sendBehavior); + noRegistered &= addConnectBehavior(transportAddress, connectBehavior); } return noRegistered; } /** - * Adds a new delegate transport that is used for communication with the given transport address. + * Adds a new connect behavior that is used for creating connections with the given transport address. * - * @return {@code true} if no other delegate was registered for this address before. + * @return {@code true} if no other send behavior was registered for this address before. */ - public boolean addDelegate(TransportAddress transportAddress, DelegateTransport transport) { - return transport().transports.put(transportAddress, transport) == null; + public boolean addConnectBehavior(TransportAddress transportAddress, ConnectBehavior connectBehavior) { + return transport().addConnectBehavior(transportAddress, connectBehavior); } /** - * Adds a new send behavior that is used for communication with the given transport address. + * Adds a new get connection behavior that is used for communication with the given transport service. * - * @return {@code true} if no other send behavior was registered for this address before. + * @return {@code true} if no other get connection behavior was registered for any of the addresses bound by transport service. */ - public boolean addSendBehavior(TransportAddress transportAddress, SendRequestBehavior sendBehavior) { - return transport().addSendBehavior(transportAddress, sendBehavior); + public boolean addGetConnectionBehavior(TransportService transportService, GetConnectionBehavior behavior) { + boolean noRegistered = true; + for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { + noRegistered &= addGetConnectionBehavior(transportAddress, behavior); + } + return noRegistered; } - private LookupTestTransport transport() { + /** + * Adds a get connection behavior that is used for communication with the given transport address. + * + * @return {@code true} if no other get connection behavior was registered for this address before. + */ + public boolean addGetConnectionBehavior(TransportAddress transportAddress, GetConnectionBehavior behavior) { + return connectionManager().addConnectBehavior(transportAddress, behavior); + } + + /** + * Adds a new get connection behavior that is used for communication with the given transport service. + * + * @return {@code true} if no other node connected behavior was registered for any of the addresses bound by transport service. + */ + public boolean addNodeConnectedBehavior(TransportService transportService, NodeConnectedBehavior behavior) { + boolean noRegistered = true; + for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { + noRegistered &= addNodeConnectedBehavior(transportAddress, behavior); + } + return noRegistered; + } + + /** + * Adds a get connection behavior that is used for communication with the given transport address. + * + * @return {@code true} if no other node connected behavior was registered for this address before. + */ + public boolean addNodeConnectedBehavior(TransportAddress transportAddress, NodeConnectedBehavior behavior) { + return connectionManager().addNodeConnectedBehavior(transportAddress, behavior); + } + + /** + * Adds a new delegate transport that is used for communication with the given transport address. + * + * @return {@code true} if no other delegate was registered for this address before. + */ + public boolean addDelegate(TransportAddress transportAddress, DelegateTransport transport) { + return transport().transports.put(transportAddress, transport) == null; + } + + public LookupTestTransport transport() { return (LookupTestTransport) transport; } + public MockConnectionManager connectionManager() { + return (MockConnectionManager) connectionManager; + } + + private static class MockConnectionManager extends ConnectionManager { + + private final ConnectionManager delegate; + private final ConcurrentHashMap getConnectionBehaviors = new ConcurrentHashMap<>(); + private final ConcurrentHashMap nodeConnectedBehaviors = new ConcurrentHashMap<>(); + private volatile GetConnectionBehavior defaultGetConnectionBehavior = null; + + private MockConnectionManager(ConnectionManager delegate, Settings settings, Transport transport, ThreadPool threadPool) { + super(settings, transport, threadPool); + this.delegate = delegate; + } + + boolean addConnectBehavior(TransportAddress transportAddress, GetConnectionBehavior connectBehavior) { + return getConnectionBehaviors.put(transportAddress, connectBehavior) == null; + } + + boolean addNodeConnectedBehavior(TransportAddress transportAddress, NodeConnectedBehavior behavior) { + return nodeConnectedBehaviors.put(transportAddress, behavior) == null; + } + + void clearBehaviors() { + getConnectionBehaviors.clear(); + nodeConnectedBehaviors.clear(); + } + + void clearBehavior(TransportAddress transportAddress) { + getConnectionBehaviors.remove(transportAddress); + nodeConnectedBehaviors.remove(transportAddress); + } + + @Override + public Transport.Connection getConnection(DiscoveryNode node) { + TransportAddress address = node.getAddress(); + GetConnectionBehavior behavior = getConnectionBehaviors.getOrDefault(address, defaultGetConnectionBehavior); + if (behavior == null) { + return delegate.getConnection(node); + } else { + return behavior.getConnection(delegate, node); + } + } + + public boolean nodeConnected(DiscoveryNode node) { + TransportAddress address = node.getAddress(); + NodeConnectedBehavior behavior = nodeConnectedBehaviors.get(address); + if (behavior == null) { + return delegate.nodeConnected(node); + } else { + return behavior.nodeConnected(delegate, node); + } + } + + @Override + public void addListener(TransportConnectionListener listener) { + delegate.addListener(listener); + } + + @Override + public void removeListener(TransportConnectionListener listener) { + delegate.removeListener(listener); + } + + public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, + CheckedBiConsumer connectionValidator) + throws ConnectTransportException { + delegate.connectToNode(node, connectionProfile, connectionValidator); + } + + public void disconnectFromNode(DiscoveryNode node) { + delegate.disconnectFromNode(node); + } + + public int connectedNodeCount() { + return delegate.connectedNodeCount(); + } + + @Override + public void close() { + delegate.close(); + } + } + /** * A lookup transport that has a list of potential Transport implementations to delegate to for node operations, * if none is registered, then the default one is used. @@ -457,7 +595,9 @@ private static class LookupTestTransport extends DelegateTransport { private final ConcurrentMap transports = ConcurrentCollections.newConcurrentMap(); private final ConcurrentHashMap sendBehaviors = new ConcurrentHashMap<>(); + private final ConcurrentHashMap connectBehaviors = new ConcurrentHashMap<>(); private volatile MockTransportService.SendRequestBehavior defaultSendRequest = null; + private volatile MockTransportService.ConnectBehavior defaultConnectBehavior = null; private LookupTestTransport(Transport transport) { super(transport); @@ -465,23 +605,37 @@ private LookupTestTransport(Transport transport) { @Override public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - Transport transportToUse = transports.getOrDefault(node.getAddress(), transport); - return new WrappedConnection(transportToUse.openConnection(node, profile)); + TransportAddress address = node.getAddress(); + MockTransportService.ConnectBehavior behavior = connectBehaviors.getOrDefault(address, defaultConnectBehavior); + Connection connection; + if (behavior == null) { + connection = transport.openConnection(node, profile); + } else { + connection = behavior.openConnection(transport, node, profile); + } + + return new WrappedConnection(connection); } boolean addSendBehavior(TransportAddress transportAddress, MockTransportService.SendRequestBehavior sendBehavior) { return sendBehaviors.put(transportAddress, sendBehavior) == null; } - void clearSendBehaviors() { + boolean addConnectBehavior(TransportAddress transportAddress, MockTransportService.ConnectBehavior connectBehavior) { + return connectBehaviors.put(transportAddress, connectBehavior) == null; + } + + void clearBehaviors() { sendBehaviors.clear(); + connectBehaviors.clear(); } - void clearSendBehavior(TransportAddress transportAddress) { + void clearBehavior(TransportAddress transportAddress) { MockTransportService.SendRequestBehavior behavior = sendBehaviors.remove(transportAddress); if (behavior != null) { behavior.clearCallback(); } + connectBehaviors.remove(transportAddress); } private class WrappedConnection implements Transport.Connection { @@ -819,6 +973,21 @@ public interface SendRequestBehavior { void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException; - default void clearCallback() {}; + default void clearCallback() {} + } + + @FunctionalInterface + public interface ConnectBehavior { + Transport.Connection openConnection(Transport transport, DiscoveryNode discoveryNode, ConnectionProfile profile); + } + + @FunctionalInterface + public interface GetConnectionBehavior { + Transport.Connection getConnection(ConnectionManager connectionManager, DiscoveryNode discoveryNode); + } + + @FunctionalInterface + public interface NodeConnectedBehavior { + boolean nodeConnected(ConnectionManager connectionManager, DiscoveryNode discoveryNode); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/NewCapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/NewCapturingTransport.java new file mode 100644 index 0000000000000..b90be8b1ab9a5 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/NewCapturingTransport.java @@ -0,0 +1,185 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.transport; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.transport.RemoteTransportException; +import org.elasticsearch.transport.SendRequestTransportException; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportResponse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static org.apache.lucene.util.LuceneTestCase.rarely; + +public class NewCapturingTransport { + + public static class CapturedRequest { + + public final DiscoveryNode node; + public final long requestId; + public final String action; + public final TransportRequest request; + public CapturedRequest(DiscoveryNode node, long requestId, String action, TransportRequest request) { + this.node = node; + this.requestId = requestId; + this.action = action; + this.request = request; + } + + } + private final MockTransportService mockTransportService; + private final Transport transport; + + private final ConcurrentMap> requests = new ConcurrentHashMap<>(); + private final BlockingQueue capturedRequests = ConcurrentCollections.newBlockingQueue(); + + public NewCapturingTransport(MockTransportService mockTransportService) { + this.mockTransportService = mockTransportService; + this.transport = mockTransportService.transport(); + } + + /** returns all requests captured so far. Doesn't clear the captured request list. See {@link #clear()} */ + public CapturingTransport.CapturedRequest[] capturedRequests() { + return capturedRequests.toArray(new CapturingTransport.CapturedRequest[0]); + } + + /** + * Returns all requests captured so far. This method does clear the + * captured requests list. If you do not want the captured requests + * list cleared, use {@link #capturedRequests()}. + * + * @return the captured requests + */ + public CapturingTransport.CapturedRequest[] getCapturedRequestsAndClear() { + List requests = new ArrayList<>(capturedRequests.size()); + capturedRequests.drainTo(requests); + return requests.toArray(new CapturingTransport.CapturedRequest[0]); + } + + private Map> groupRequestsByTargetNode(Collection requests) { + Map> result = new HashMap<>(); + for (CapturingTransport.CapturedRequest request : requests) { + result.computeIfAbsent(request.node.getId(), node -> new ArrayList<>()).add(request); + } + return result; + } + + /** + * returns all requests captured so far, grouped by target node. + * Doesn't clear the captured request list. See {@link #clear()} + */ + public Map> capturedRequestsByTargetNode() { + return groupRequestsByTargetNode(capturedRequests); + } + + /** + * Returns all requests captured so far, grouped by target node. + * This method does clear the captured request list. If you do not + * want the captured requests list cleared, use + * {@link #capturedRequestsByTargetNode()}. + * + * @return the captured requests grouped by target node + */ + public Map> getCapturedRequestsByTargetNodeAndClear() { + List requests = new ArrayList<>(capturedRequests.size()); + capturedRequests.drainTo(requests); + return groupRequestsByTargetNode(requests); + } + + /** clears captured requests */ + public void clear() { + capturedRequests.clear(); + } + + /** simulate a response for the given requestId */ + public void handleResponse(final long requestId, final TransportResponse response) { + // TODO: Null listener + transport.getResponseHandlers().onResponseReceived(requestId, null).handleResponse(response); + } + + /** + * simulate a local error for the given requestId, will be wrapped + * by a {@link SendRequestTransportException} + * + * @param requestId the id corresponding to the captured send + * request + * @param t the failure to wrap + */ + public void handleLocalError(final long requestId, final Throwable t) { + Tuple request = requests.get(requestId); + assert request != null; + this.handleError(requestId, new SendRequestTransportException(request.v1(), request.v2(), t)); + } + + /** + * simulate a remote error for the given requestId, will be wrapped + * by a {@link RemoteTransportException} + * + * @param requestId the id corresponding to the captured send + * request + * @param t the failure to wrap + */ + public void handleRemoteError(final long requestId, final Throwable t) { + final RemoteTransportException remoteException; + if (rarely(Randomness.get())) { + remoteException = new RemoteTransportException("remote failure, coming from local node", t); + } else { + try (BytesStreamOutput output = new BytesStreamOutput()) { + output.writeException(t); + remoteException = new RemoteTransportException("remote failure", output.bytes().streamInput().readException()); + } catch (IOException ioException) { + throw new ElasticsearchException("failed to serialize/deserialize supplied exception " + t, ioException); + } + } + this.handleError(requestId, remoteException); + } + + /** + * simulate an error for the given requestId, unlike + * {@link #handleLocalError(long, Throwable)} and + * {@link #handleRemoteError(long, Throwable)}, the provided + * exception will not be wrapped but will be delivered to the + * transport layer as is + * + * @param requestId the id corresponding to the captured send + * request + * @param e the failure + */ + public void handleError(final long requestId, final TransportException e) { + // TODO: Null listener + transport.getResponseHandlers().onResponseReceived(requestId, null).handleException(e); + } +} From 510685243200fd6329bf5dc0dbfd387d8954b1b8 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 26 Jul 2018 18:17:02 -0600 Subject: [PATCH 17/29] Work on fixing tests --- .../TransportBroadcastByNodeActionTests.java | 5 ++- .../TransportMasterNodeActionTests.java | 6 ++- .../nodes/TransportNodesActionTests.java | 6 ++- .../GlobalCheckpointSyncActionTests.java | 3 +- .../test/transport/CapturingTransport.java | 32 +++++++++++++-- .../test/transport/MockTransportService.java | 40 +++++++++++++++++-- 6 files changed, 79 insertions(+), 13 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index fdc3d890363ad..0034e0b208f73 100644 --- a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -54,8 +54,10 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; +import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ConnectionManager; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseOptions; @@ -191,8 +193,9 @@ public void setUp() throws Exception { super.setUp(); transport = new CapturingTransport(); clusterService = createClusterService(THREAD_POOL); - final TransportService transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL, + MockTransportService transportService = new MockTransportService(clusterService.getSettings(), transport, THREAD_POOL, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); + transport.bootstrapWithTransportService(transportService); transportService.start(); transportService.acceptIncomingRequests(); setClusterState(clusterService, TEST_INDEX); diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java index b27bc9ad79432..f7cbc0d73bd77 100644 --- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java @@ -46,6 +46,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; +import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectTransportException; @@ -70,7 +71,7 @@ public class TransportMasterNodeActionTests extends ESTestCase { private static ThreadPool threadPool; private ClusterService clusterService; - private TransportService transportService; + private MockTransportService transportService; private CapturingTransport transport; private DiscoveryNode localNode; private DiscoveryNode remoteNode; @@ -87,8 +88,9 @@ public void setUp() throws Exception { super.setUp(); transport = new CapturingTransport(); clusterService = createClusterService(threadPool); - transportService = new TransportService(clusterService.getSettings(), transport, threadPool, + transportService = new MockTransportService(clusterService.getSettings(), transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); + transport.bootstrapWithTransportService(transportService); transportService.start(); transportService.acceptIncomingRequests(); localNode = new DiscoveryNode("local_node", buildNewFakeTransportAddress(), Collections.emptyMap(), diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java index 8a79da044b915..9e0e5cbbd1856 100644 --- a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; +import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -64,7 +65,7 @@ public class TransportNodesActionTests extends ESTestCase { private ClusterService clusterService; private CapturingTransport transport; - private TransportService transportService; + private MockTransportService transportService; public void testRequestIsSentToEachNode() throws Exception { TransportNodesAction action = getTestTransportNodesAction(); @@ -180,8 +181,9 @@ public void setUp() throws Exception { super.setUp(); transport = new CapturingTransport(); clusterService = createClusterService(THREAD_POOL); - transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL, + transportService = new MockTransportService(clusterService.getSettings(), transport, THREAD_POOL, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); + transport.bootstrapWithTransportService(transportService); transportService.start(); transportService.acceptIncomingRequests(); int numNodes = randomIntBetween(3, 10); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java index 596575abc3025..88b82b175a81e 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; +import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; @@ -57,7 +58,7 @@ public void setUp() throws Exception { threadPool = new TestThreadPool(getClass().getName()); transport = new CapturingTransport(); clusterService = createClusterService(threadPool); - transportService = new TransportService(clusterService.getSettings(), transport, threadPool, + transportService = new MockTransportService(clusterService.getSettings(), transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java index c9f900577a8ba..fe8e92ad7e81f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.transport.ConnectionManager; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.RequestHandlerRegistry; @@ -81,6 +82,33 @@ public CapturedRequest(DiscoveryNode node, long requestId, String action, Transp private ConcurrentMap> requests = new ConcurrentHashMap<>(); private BlockingQueue capturedRequests = ConcurrentCollections.newBlockingQueue(); + public void bootstrapWithTransportService(MockTransportService transportService) { + transportService.addNodeConnectedBehavior((connectionManager, discoveryNode) -> true); + transportService.addSendBehavior((connection, requestId, action, request, options) -> { + DiscoveryNode node = connection.getNode(); + requests.put(requestId, Tuple.tuple(node, action)); + capturedRequests.add(new CapturedRequest(node, requestId, action, request)); + }); + transportService.addGetConnectionBehavior((connectionManager, discoveryNode) -> new Connection() { + @Override + public DiscoveryNode getNode() { + return discoveryNode; + } + + @Override + public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) + throws TransportException { + requests.put(requestId, Tuple.tuple(discoveryNode, action)); + capturedRequests.add(new CapturedRequest(discoveryNode, requestId, action, request)); + } + + @Override + public void close() { + + } + }); + } + /** returns all requests captured so far. Doesn't clear the captured request list. See {@link #clear()} */ public CapturedRequest[] capturedRequests() { return capturedRequests.toArray(new CapturedRequest[0]); @@ -262,10 +290,6 @@ public List getLocalAddresses() { return Collections.emptyList(); } - public Connection getConnection(DiscoveryNode node) { - return openConnection(node, null); - } - @Override public void registerRequestHandler(RequestHandlerRegistry reg) { synchronized (requestHandlerMutex) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index c3c7a348b903b..3c74a80f1cc34 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -422,6 +422,17 @@ public boolean addSendBehavior(TransportAddress transportAddress, SendRequestBeh return transport().addSendBehavior(transportAddress, sendBehavior); } + /** + * Adds a send behavior that is the default send behavior. + * + * @return {@code true} if no default send behavior was registered + */ + public boolean addSendBehavior(SendRequestBehavior behavior) { + SendRequestBehavior prior = transport().defaultSendRequest; + transport().defaultSendRequest = behavior; + return prior == null; + } + /** * Adds a new connect behavior that is used for creating connections with the given transport service. @@ -468,7 +479,18 @@ public boolean addGetConnectionBehavior(TransportAddress transportAddress, GetCo } /** - * Adds a new get connection behavior that is used for communication with the given transport service. + * Adds a get connection behavior that is the default get connection behavior. + * + * @return {@code true} if no default get connection behavior was registered. + */ + public boolean addGetConnectionBehavior(GetConnectionBehavior behavior) { + GetConnectionBehavior prior = connectionManager().defaultGetConnectionBehavior; + connectionManager().defaultGetConnectionBehavior = behavior; + return prior == null; + } + + /** + * Adds a node connected behavior that is used for the given transport service. * * @return {@code true} if no other node connected behavior was registered for any of the addresses bound by transport service. */ @@ -481,7 +503,7 @@ public boolean addNodeConnectedBehavior(TransportService transportService, NodeC } /** - * Adds a get connection behavior that is used for communication with the given transport address. + * Adds a node connected behavior that is used for the given transport address. * * @return {@code true} if no other node connected behavior was registered for this address before. */ @@ -489,6 +511,17 @@ public boolean addNodeConnectedBehavior(TransportAddress transportAddress, NodeC return connectionManager().addNodeConnectedBehavior(transportAddress, behavior); } + /** + * Adds a node connected behavior that is the default node connected behavior. + * + * @return {@code true} if no default node connected behavior was registered. + */ + public boolean addNodeConnectedBehavior(NodeConnectedBehavior behavior) { + NodeConnectedBehavior prior = connectionManager().defaultNodeConnectedBehavior; + connectionManager().defaultNodeConnectedBehavior = behavior; + return prior == null; + } + /** * Adds a new delegate transport that is used for communication with the given transport address. * @@ -512,6 +545,7 @@ private static class MockConnectionManager extends ConnectionManager { private final ConcurrentHashMap getConnectionBehaviors = new ConcurrentHashMap<>(); private final ConcurrentHashMap nodeConnectedBehaviors = new ConcurrentHashMap<>(); private volatile GetConnectionBehavior defaultGetConnectionBehavior = null; + private volatile NodeConnectedBehavior defaultNodeConnectedBehavior = null; private MockConnectionManager(ConnectionManager delegate, Settings settings, Transport transport, ThreadPool threadPool) { super(settings, transport, threadPool); @@ -549,7 +583,7 @@ public Transport.Connection getConnection(DiscoveryNode node) { public boolean nodeConnected(DiscoveryNode node) { TransportAddress address = node.getAddress(); - NodeConnectedBehavior behavior = nodeConnectedBehaviors.get(address); + NodeConnectedBehavior behavior = nodeConnectedBehaviors.getOrDefault(address, defaultNodeConnectedBehavior); if (behavior == null) { return delegate.nodeConnected(node); } else { From af37b63111aee65368806ed590e128d83df1b1e8 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 27 Jul 2018 15:35:03 -0600 Subject: [PATCH 18/29] Fix tests --- .../indices/get/GetIndexActionTests.java | 4 +- .../settings/get/GetSettingsActionTests.java | 2 +- .../action/bulk/TransportBulkActionTests.java | 4 +- .../bulk/TransportBulkActionTookTests.java | 4 +- .../TransportBroadcastByNodeActionTests.java | 5 +- .../TransportMasterNodeActionTests.java | 8 +- .../nodes/TransportNodesActionTests.java | 6 +- .../TransportReplicationActionTests.java | 2 +- .../TransportWriteActionTests.java | 4 +- ...ortInstanceSingleOperationActionTests.java | 8 +- .../TransportClientNodesServiceTests.java | 2 - .../action/shard/ShardStateActionTests.java | 4 +- .../health/ClusterStateHealthTests.java | 3 +- .../GlobalCheckpointSyncActionTests.java | 8 +- .../test/transport/CapturingTransport.java | 26 ++- .../test/transport/MockConnectionManager.java | 130 ++++++++++++ .../test/transport/MockTransportService.java | 91 +-------- .../test/transport/NewCapturingTransport.java | 185 ------------------ 18 files changed, 175 insertions(+), 321 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/transport/NewCapturingTransport.java diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java index d8b9a275be13d..b67c2e2954d04 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java @@ -36,8 +36,6 @@ import org.elasticsearch.test.transport.CapturingTransport; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TcpTransport; -import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import org.junit.After; import org.junit.Before; @@ -65,7 +63,7 @@ public void setUp() throws Exception { clusterService = getInstanceFromNode(ClusterService.class); indicesService = getInstanceFromNode(IndicesService.class); CapturingTransport capturingTransport = new CapturingTransport(); - transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool, + transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java index 11f0188c8c0b0..03ccebba10dbd 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java @@ -75,7 +75,7 @@ public void setUp() throws Exception { threadPool = new TestThreadPool("GetSettingsActionTests"); clusterService = createClusterService(threadPool); CapturingTransport capturingTransport = new CapturingTransport(); - transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool, + transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 3bd66af1bab05..a1abd4d61f7fd 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -80,8 +80,8 @@ public void setUp() throws Exception { threadPool = new TestThreadPool("TransportBulkActionTookTests"); clusterService = createClusterService(threadPool); CapturingTransport capturingTransport = new CapturingTransport(); - transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool, - TransportService.NOOP_TRANSPORT_INTERCEPTOR, + transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 9d5193180299d..f6559f226417e 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -92,8 +92,8 @@ public void tearDown() throws Exception { private TransportBulkAction createAction(boolean controlled, AtomicLong expected) { CapturingTransport capturingTransport = new CapturingTransport(); - TransportService transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool, - TransportService.NOOP_TRANSPORT_INTERCEPTOR, + TransportService transportService = capturingTransport.createCapturingTransportService(clusterService.getSettings(), threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index 0034e0b208f73..d12c3ca26eb08 100644 --- a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -54,10 +54,8 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; -import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.ConnectionManager; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseOptions; @@ -193,9 +191,8 @@ public void setUp() throws Exception { super.setUp(); transport = new CapturingTransport(); clusterService = createClusterService(THREAD_POOL); - MockTransportService transportService = new MockTransportService(clusterService.getSettings(), transport, THREAD_POOL, + TransportService transportService = transport.createCapturingTransportService(clusterService.getSettings(), THREAD_POOL, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); - transport.bootstrapWithTransportService(transportService); transportService.start(); transportService.acceptIncomingRequests(); setClusterState(clusterService, TEST_INDEX); diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java index f7cbc0d73bd77..f592b3d803af5 100644 --- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java @@ -46,7 +46,6 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; -import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectTransportException; @@ -71,7 +70,7 @@ public class TransportMasterNodeActionTests extends ESTestCase { private static ThreadPool threadPool; private ClusterService clusterService; - private MockTransportService transportService; + private TransportService transportService; private CapturingTransport transport; private DiscoveryNode localNode; private DiscoveryNode remoteNode; @@ -88,9 +87,8 @@ public void setUp() throws Exception { super.setUp(); transport = new CapturingTransport(); clusterService = createClusterService(threadPool); - transportService = new MockTransportService(clusterService.getSettings(), transport, threadPool, - TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); - transport.bootstrapWithTransportService(transportService); + transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); localNode = new DiscoveryNode("local_node", buildNewFakeTransportAddress(), Collections.emptyMap(), diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java index 9e0e5cbbd1856..be7ec477ff48c 100644 --- a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; -import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -65,7 +64,7 @@ public class TransportNodesActionTests extends ESTestCase { private ClusterService clusterService; private CapturingTransport transport; - private MockTransportService transportService; + private TransportService transportService; public void testRequestIsSentToEachNode() throws Exception { TransportNodesAction action = getTestTransportNodesAction(); @@ -181,9 +180,8 @@ public void setUp() throws Exception { super.setUp(); transport = new CapturingTransport(); clusterService = createClusterService(THREAD_POOL); - transportService = new MockTransportService(clusterService.getSettings(), transport, THREAD_POOL, + transportService = transport.createCapturingTransportService(clusterService.getSettings(), THREAD_POOL, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); - transport.bootstrapWithTransportService(transportService); transportService.start(); transportService.acceptIncomingRequests(); int numNodes = randomIntBetween(3, 10); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 08301e99d6a69..6c429aff49801 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -163,7 +163,7 @@ public void setUp() throws Exception { super.setUp(); transport = new CapturingTransport(); clusterService = createClusterService(threadPool); - transportService = new TransportService(clusterService.getSettings(), transport, threadPool, + transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index bfcc5938a8690..9c0dd2d867a38 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -255,8 +255,8 @@ public void testDocumentFailureInShardOperationOnReplica() throws Exception { public void testReplicaProxy() throws InterruptedException, ExecutionException { CapturingTransport transport = new CapturingTransport(); - TransportService transportService = new TransportService(clusterService.getSettings(), transport, threadPool, - TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); + TransportService transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); ShardStateAction shardStateAction = new ShardStateAction(Settings.EMPTY, clusterService, transportService, null, null, threadPool); diff --git a/server/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java index 8db45cc5508ef..0505143592497 100644 --- a/server/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; @@ -144,16 +143,15 @@ public void setUp() throws Exception { super.setUp(); transport = new CapturingTransport(); clusterService = createClusterService(THREAD_POOL); - transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL, - TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet() - ); + transportService = transport.createCapturingTransportService(clusterService.getSettings(), THREAD_POOL, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); action = new TestTransportInstanceSingleOperationAction( Settings.EMPTY, "indices:admin/test", transportService, - new ActionFilters(new HashSet()), + new ActionFilters(new HashSet<>()), new MyResolver(), Request::new ); diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 038296793295d..1c3fda05825dc 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -41,8 +41,6 @@ import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.ConnectionManager; -import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java index 1d78cdeb98374..64fa51d159a54 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java @@ -124,8 +124,8 @@ public void setUp() throws Exception { super.setUp(); this.transport = new CapturingTransport(); clusterService = createClusterService(THREAD_POOL); - transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL, - TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); + transportService = transport.createCapturingTransportService(clusterService.getSettings(), THREAD_POOL, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); shardStateAction = new TestShardStateAction(Settings.EMPTY, clusterService, transportService, null, null); diff --git a/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java index 0f5f4870ae1bb..32a8c06bb7020 100644 --- a/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java @@ -94,7 +94,8 @@ public static void setupThreadPool() { public void setUp() throws Exception { super.setUp(); clusterService = createClusterService(threadPool); - transportService = new TransportService(clusterService.getSettings(), new CapturingTransport(), threadPool, + CapturingTransport transport = new CapturingTransport(); + transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java index 88b82b175a81e..0888dfd3c40c5 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java @@ -17,12 +17,12 @@ package org.elasticsearch.index.seqno; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.IndexShard; @@ -31,10 +31,8 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; -import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import java.util.Collections; @@ -48,7 +46,7 @@ public class GlobalCheckpointSyncActionTests extends ESTestCase { private ThreadPool threadPool; - private Transport transport; + private CapturingTransport transport; private ClusterService clusterService; private TransportService transportService; private ShardStateAction shardStateAction; @@ -58,7 +56,7 @@ public void setUp() throws Exception { threadPool = new TestThreadPool(getClass().getName()); transport = new CapturingTransport(); clusterService = createClusterService(threadPool); - transportService = new MockTransportService(clusterService.getSettings(), transport, threadPool, + transportService = transport.createCapturingTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, Collections.emptySet()); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java index fe8e92ad7e81f..69e5ffbd13397 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java @@ -21,15 +21,19 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectionManager; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.RemoteTransportException; @@ -38,9 +42,11 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportConnectionListener; import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportInterceptor; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportStats; import java.io.IOException; @@ -51,9 +57,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; import static org.apache.lucene.util.LuceneTestCase.rarely; @@ -82,14 +90,13 @@ public CapturedRequest(DiscoveryNode node, long requestId, String action, Transp private ConcurrentMap> requests = new ConcurrentHashMap<>(); private BlockingQueue capturedRequests = ConcurrentCollections.newBlockingQueue(); - public void bootstrapWithTransportService(MockTransportService transportService) { - transportService.addNodeConnectedBehavior((connectionManager, discoveryNode) -> true); - transportService.addSendBehavior((connection, requestId, action, request, options) -> { - DiscoveryNode node = connection.getNode(); - requests.put(requestId, Tuple.tuple(node, action)); - capturedRequests.add(new CapturedRequest(node, requestId, action, request)); - }); - transportService.addGetConnectionBehavior((connectionManager, discoveryNode) -> new Connection() { + public TransportService createCapturingTransportService(Settings settings, ThreadPool threadPool, TransportInterceptor interceptor, + Function localNodeFactory, + @Nullable ClusterSettings clusterSettings, Set taskHeaders) { + MockConnectionManager connectionManager = new MockConnectionManager(new ConnectionManager(settings, this, threadPool), settings, + this, threadPool); + connectionManager.setDefaultNodeConnectedBehavior((cm, discoveryNode) -> true); + connectionManager.setDefaultConnectBehavior((cm, discoveryNode) -> new Connection() { @Override public DiscoveryNode getNode() { return discoveryNode; @@ -107,6 +114,9 @@ public void close() { } }); + return new TransportService(settings, this, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders, + connectionManager); + } /** returns all requests captured so far. Doesn't clear the captured request list. See {@link #clear()} */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java new file mode 100644 index 0000000000000..4f21655c11d4e --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test.transport; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.ConnectionManager; +import org.elasticsearch.transport.ConnectionProfile; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportConnectionListener; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +class MockConnectionManager extends ConnectionManager { + + private final ConnectionManager delegate; + private final ConcurrentMap getConnectionBehaviors; + private final ConcurrentMap nodeConnectedBehaviors; + private volatile MockTransportService.GetConnectionBehavior defaultGetConnectionBehavior = null; + private volatile MockTransportService.NodeConnectedBehavior defaultNodeConnectedBehavior = null; + + MockConnectionManager(ConnectionManager delegate, Settings settings, Transport transport, ThreadPool threadPool) { + super(settings, transport, threadPool); + this.delegate = delegate; + this.getConnectionBehaviors = new ConcurrentHashMap<>(); + this.nodeConnectedBehaviors = new ConcurrentHashMap<>(); + } + + boolean addConnectBehavior(TransportAddress transportAddress, MockTransportService.GetConnectionBehavior connectBehavior) { + return getConnectionBehaviors.put(transportAddress, connectBehavior) == null; + } + + boolean setDefaultConnectBehavior(MockTransportService.GetConnectionBehavior behavior) { + MockTransportService.GetConnectionBehavior prior = defaultGetConnectionBehavior; + defaultGetConnectionBehavior = behavior; + return prior == null; + } + + boolean addNodeConnectedBehavior(TransportAddress transportAddress, MockTransportService.NodeConnectedBehavior behavior) { + return nodeConnectedBehaviors.put(transportAddress, behavior) == null; + } + + boolean setDefaultNodeConnectedBehavior(MockTransportService.NodeConnectedBehavior behavior) { + MockTransportService.NodeConnectedBehavior prior = defaultNodeConnectedBehavior; + defaultNodeConnectedBehavior = behavior; + return prior == null; + } + + void clearBehaviors() { + getConnectionBehaviors.clear(); + nodeConnectedBehaviors.clear(); + } + + void clearBehavior(TransportAddress transportAddress) { + getConnectionBehaviors.remove(transportAddress); + nodeConnectedBehaviors.remove(transportAddress); + } + + @Override + public Transport.Connection getConnection(DiscoveryNode node) { + TransportAddress address = node.getAddress(); + MockTransportService.GetConnectionBehavior behavior = getConnectionBehaviors.getOrDefault(address, defaultGetConnectionBehavior); + if (behavior == null) { + return delegate.getConnection(node); + } else { + return behavior.getConnection(delegate, node); + } + } + + public boolean nodeConnected(DiscoveryNode node) { + TransportAddress address = node.getAddress(); + MockTransportService.NodeConnectedBehavior behavior = nodeConnectedBehaviors.getOrDefault(address, defaultNodeConnectedBehavior); + if (behavior == null) { + return delegate.nodeConnected(node); + } else { + return behavior.nodeConnected(delegate, node); + } + } + + @Override + public void addListener(TransportConnectionListener listener) { + delegate.addListener(listener); + } + + @Override + public void removeListener(TransportConnectionListener listener) { + delegate.removeListener(listener); + } + + public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, + CheckedBiConsumer connectionValidator) + throws ConnectTransportException { + delegate.connectToNode(node, connectionProfile, connectionValidator); + } + + public void disconnectFromNode(DiscoveryNode node) { + delegate.disconnectFromNode(node); + } + + public int connectedNodeCount() { + return delegate.connectedNodeCount(); + } + + @Override + public void close() { + delegate.close(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 3c74a80f1cc34..86f94ebbde30b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.component.Lifecycle; @@ -484,9 +483,7 @@ public boolean addGetConnectionBehavior(TransportAddress transportAddress, GetCo * @return {@code true} if no default get connection behavior was registered. */ public boolean addGetConnectionBehavior(GetConnectionBehavior behavior) { - GetConnectionBehavior prior = connectionManager().defaultGetConnectionBehavior; - connectionManager().defaultGetConnectionBehavior = behavior; - return prior == null; + return connectionManager().setDefaultConnectBehavior(behavior); } /** @@ -517,9 +514,7 @@ public boolean addNodeConnectedBehavior(TransportAddress transportAddress, NodeC * @return {@code true} if no default node connected behavior was registered. */ public boolean addNodeConnectedBehavior(NodeConnectedBehavior behavior) { - NodeConnectedBehavior prior = connectionManager().defaultNodeConnectedBehavior; - connectionManager().defaultNodeConnectedBehavior = behavior; - return prior == null; + return connectionManager().setDefaultNodeConnectedBehavior(behavior); } /** @@ -539,88 +534,6 @@ public MockConnectionManager connectionManager() { return (MockConnectionManager) connectionManager; } - private static class MockConnectionManager extends ConnectionManager { - - private final ConnectionManager delegate; - private final ConcurrentHashMap getConnectionBehaviors = new ConcurrentHashMap<>(); - private final ConcurrentHashMap nodeConnectedBehaviors = new ConcurrentHashMap<>(); - private volatile GetConnectionBehavior defaultGetConnectionBehavior = null; - private volatile NodeConnectedBehavior defaultNodeConnectedBehavior = null; - - private MockConnectionManager(ConnectionManager delegate, Settings settings, Transport transport, ThreadPool threadPool) { - super(settings, transport, threadPool); - this.delegate = delegate; - } - - boolean addConnectBehavior(TransportAddress transportAddress, GetConnectionBehavior connectBehavior) { - return getConnectionBehaviors.put(transportAddress, connectBehavior) == null; - } - - boolean addNodeConnectedBehavior(TransportAddress transportAddress, NodeConnectedBehavior behavior) { - return nodeConnectedBehaviors.put(transportAddress, behavior) == null; - } - - void clearBehaviors() { - getConnectionBehaviors.clear(); - nodeConnectedBehaviors.clear(); - } - - void clearBehavior(TransportAddress transportAddress) { - getConnectionBehaviors.remove(transportAddress); - nodeConnectedBehaviors.remove(transportAddress); - } - - @Override - public Transport.Connection getConnection(DiscoveryNode node) { - TransportAddress address = node.getAddress(); - GetConnectionBehavior behavior = getConnectionBehaviors.getOrDefault(address, defaultGetConnectionBehavior); - if (behavior == null) { - return delegate.getConnection(node); - } else { - return behavior.getConnection(delegate, node); - } - } - - public boolean nodeConnected(DiscoveryNode node) { - TransportAddress address = node.getAddress(); - NodeConnectedBehavior behavior = nodeConnectedBehaviors.getOrDefault(address, defaultNodeConnectedBehavior); - if (behavior == null) { - return delegate.nodeConnected(node); - } else { - return behavior.nodeConnected(delegate, node); - } - } - - @Override - public void addListener(TransportConnectionListener listener) { - delegate.addListener(listener); - } - - @Override - public void removeListener(TransportConnectionListener listener) { - delegate.removeListener(listener); - } - - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - delegate.connectToNode(node, connectionProfile, connectionValidator); - } - - public void disconnectFromNode(DiscoveryNode node) { - delegate.disconnectFromNode(node); - } - - public int connectedNodeCount() { - return delegate.connectedNodeCount(); - } - - @Override - public void close() { - delegate.close(); - } - } - /** * A lookup transport that has a list of potential Transport implementations to delegate to for node operations, * if none is registered, then the default one is used. diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/NewCapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/NewCapturingTransport.java deleted file mode 100644 index b90be8b1ab9a5..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/NewCapturingTransport.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.transport; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.Randomness; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.transport.RemoteTransportException; -import org.elasticsearch.transport.SendRequestTransportException; -import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportException; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportResponse; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import static org.apache.lucene.util.LuceneTestCase.rarely; - -public class NewCapturingTransport { - - public static class CapturedRequest { - - public final DiscoveryNode node; - public final long requestId; - public final String action; - public final TransportRequest request; - public CapturedRequest(DiscoveryNode node, long requestId, String action, TransportRequest request) { - this.node = node; - this.requestId = requestId; - this.action = action; - this.request = request; - } - - } - private final MockTransportService mockTransportService; - private final Transport transport; - - private final ConcurrentMap> requests = new ConcurrentHashMap<>(); - private final BlockingQueue capturedRequests = ConcurrentCollections.newBlockingQueue(); - - public NewCapturingTransport(MockTransportService mockTransportService) { - this.mockTransportService = mockTransportService; - this.transport = mockTransportService.transport(); - } - - /** returns all requests captured so far. Doesn't clear the captured request list. See {@link #clear()} */ - public CapturingTransport.CapturedRequest[] capturedRequests() { - return capturedRequests.toArray(new CapturingTransport.CapturedRequest[0]); - } - - /** - * Returns all requests captured so far. This method does clear the - * captured requests list. If you do not want the captured requests - * list cleared, use {@link #capturedRequests()}. - * - * @return the captured requests - */ - public CapturingTransport.CapturedRequest[] getCapturedRequestsAndClear() { - List requests = new ArrayList<>(capturedRequests.size()); - capturedRequests.drainTo(requests); - return requests.toArray(new CapturingTransport.CapturedRequest[0]); - } - - private Map> groupRequestsByTargetNode(Collection requests) { - Map> result = new HashMap<>(); - for (CapturingTransport.CapturedRequest request : requests) { - result.computeIfAbsent(request.node.getId(), node -> new ArrayList<>()).add(request); - } - return result; - } - - /** - * returns all requests captured so far, grouped by target node. - * Doesn't clear the captured request list. See {@link #clear()} - */ - public Map> capturedRequestsByTargetNode() { - return groupRequestsByTargetNode(capturedRequests); - } - - /** - * Returns all requests captured so far, grouped by target node. - * This method does clear the captured request list. If you do not - * want the captured requests list cleared, use - * {@link #capturedRequestsByTargetNode()}. - * - * @return the captured requests grouped by target node - */ - public Map> getCapturedRequestsByTargetNodeAndClear() { - List requests = new ArrayList<>(capturedRequests.size()); - capturedRequests.drainTo(requests); - return groupRequestsByTargetNode(requests); - } - - /** clears captured requests */ - public void clear() { - capturedRequests.clear(); - } - - /** simulate a response for the given requestId */ - public void handleResponse(final long requestId, final TransportResponse response) { - // TODO: Null listener - transport.getResponseHandlers().onResponseReceived(requestId, null).handleResponse(response); - } - - /** - * simulate a local error for the given requestId, will be wrapped - * by a {@link SendRequestTransportException} - * - * @param requestId the id corresponding to the captured send - * request - * @param t the failure to wrap - */ - public void handleLocalError(final long requestId, final Throwable t) { - Tuple request = requests.get(requestId); - assert request != null; - this.handleError(requestId, new SendRequestTransportException(request.v1(), request.v2(), t)); - } - - /** - * simulate a remote error for the given requestId, will be wrapped - * by a {@link RemoteTransportException} - * - * @param requestId the id corresponding to the captured send - * request - * @param t the failure to wrap - */ - public void handleRemoteError(final long requestId, final Throwable t) { - final RemoteTransportException remoteException; - if (rarely(Randomness.get())) { - remoteException = new RemoteTransportException("remote failure, coming from local node", t); - } else { - try (BytesStreamOutput output = new BytesStreamOutput()) { - output.writeException(t); - remoteException = new RemoteTransportException("remote failure", output.bytes().streamInput().readException()); - } catch (IOException ioException) { - throw new ElasticsearchException("failed to serialize/deserialize supplied exception " + t, ioException); - } - } - this.handleError(requestId, remoteException); - } - - /** - * simulate an error for the given requestId, unlike - * {@link #handleLocalError(long, Throwable)} and - * {@link #handleRemoteError(long, Throwable)}, the provided - * exception will not be wrapped but will be delivered to the - * transport layer as is - * - * @param requestId the id corresponding to the captured send - * request - * @param e the failure - */ - public void handleError(final long requestId, final TransportException e) { - // TODO: Null listener - transport.getResponseHandlers().onResponseReceived(requestId, null).handleException(e); - } -} From 0436eb05d88512655737042045be9feddc069b49 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 27 Jul 2018 16:51:35 -0600 Subject: [PATCH 19/29] Work on transport client nodes service tests --- .../TransportClientNodesServiceTests.java | 21 ++++++++++++------- .../test/transport/MockConnectionManager.java | 16 +++++++------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 1c3fda05825dc..2b3374f8b7e46 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -50,7 +50,6 @@ import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseHandler; -import org.elasticsearch.transport.TransportService; import org.hamcrest.CustomMatcher; import java.io.Closeable; @@ -82,7 +81,7 @@ public class TransportClientNodesServiceTests extends ESTestCase { private static class TestIteration implements Closeable { private final ThreadPool threadPool; private final FailAndRetryMockTransport transport; - private final TransportService transportService; + private final MockTransportService transportService; private final TransportClientNodesService transportClientNodesService; private final int listNodesCount; private final int sniffNodesCount; @@ -142,7 +141,8 @@ protected ClusterState getMockClusterState(DiscoveryNode node) { return ClusterState.builder(clusterName).nodes(TestIteration.this.nodeMap.get(node.getAddress())).build(); } }; - transportService = new TransportService(settings, transport, threadPool, new TransportInterceptor() { + + transportService = new MockTransportService(settings, transport, threadPool, new TransportInterceptor() { @Override public AsyncSender interceptSender(AsyncSender sender) { return new AsyncSender() { @@ -164,6 +164,11 @@ public void sendRequest(Transport.Connection conne assert addr == null : "boundAddress: " + addr; return DiscoveryNode.createLocal(settings, buildNewFakeTransportAddress(), UUIDs.randomBase64UUID()); }, null, Collections.emptySet()); + transportService.addNodeConnectedBehavior((connectionManager, discoveryNode) -> false); + transportService.addGetConnectionBehavior((connectionManager, discoveryNode) -> { + // The FailAndRetryTransport does not use the connection profile + return transport.openConnection(discoveryNode, null); + }); transportService.start(); transportService.acceptIncomingRequests(); transportClientNodesService = @@ -358,17 +363,17 @@ public void testSniffNodesSamplerClosesConnections() throws Exception { final List establishedConnections = new CopyOnWriteArrayList<>(); final List reusedConnections = new CopyOnWriteArrayList<>(); + clientService.addConnectBehavior(remoteService, (transport, discoveryNode, profile) -> { + MockConnection connection = new MockConnection(transport.openConnection(discoveryNode, profile)); + establishedConnections.add(connection); + return connection; + }); clientService.addGetConnectionBehavior(remoteService, (connectionManager, discoveryNode) -> { MockConnection connection = new MockConnection(connectionManager.getConnection(discoveryNode)); reusedConnections.add(connection); return connection; }); - clientService.addConnectBehavior(remoteService, (transport, discoveryNode, profile) -> { - MockConnection connection = new MockConnection(transport.openConnection(discoveryNode, profile)); - establishedConnections.add(connection); - return connection; - }); clientService.start(); clientService.acceptIncomingRequests(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java index 4f21655c11d4e..099919af20517 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java @@ -33,7 +33,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -class MockConnectionManager extends ConnectionManager { +public class MockConnectionManager extends ConnectionManager { private final ConnectionManager delegate; private final ConcurrentMap getConnectionBehaviors; @@ -41,39 +41,39 @@ class MockConnectionManager extends ConnectionManager { private volatile MockTransportService.GetConnectionBehavior defaultGetConnectionBehavior = null; private volatile MockTransportService.NodeConnectedBehavior defaultNodeConnectedBehavior = null; - MockConnectionManager(ConnectionManager delegate, Settings settings, Transport transport, ThreadPool threadPool) { + public MockConnectionManager(ConnectionManager delegate, Settings settings, Transport transport, ThreadPool threadPool) { super(settings, transport, threadPool); this.delegate = delegate; this.getConnectionBehaviors = new ConcurrentHashMap<>(); this.nodeConnectedBehaviors = new ConcurrentHashMap<>(); } - boolean addConnectBehavior(TransportAddress transportAddress, MockTransportService.GetConnectionBehavior connectBehavior) { + public boolean addConnectBehavior(TransportAddress transportAddress, MockTransportService.GetConnectionBehavior connectBehavior) { return getConnectionBehaviors.put(transportAddress, connectBehavior) == null; } - boolean setDefaultConnectBehavior(MockTransportService.GetConnectionBehavior behavior) { + public boolean setDefaultConnectBehavior(MockTransportService.GetConnectionBehavior behavior) { MockTransportService.GetConnectionBehavior prior = defaultGetConnectionBehavior; defaultGetConnectionBehavior = behavior; return prior == null; } - boolean addNodeConnectedBehavior(TransportAddress transportAddress, MockTransportService.NodeConnectedBehavior behavior) { + public boolean addNodeConnectedBehavior(TransportAddress transportAddress, MockTransportService.NodeConnectedBehavior behavior) { return nodeConnectedBehaviors.put(transportAddress, behavior) == null; } - boolean setDefaultNodeConnectedBehavior(MockTransportService.NodeConnectedBehavior behavior) { + public boolean setDefaultNodeConnectedBehavior(MockTransportService.NodeConnectedBehavior behavior) { MockTransportService.NodeConnectedBehavior prior = defaultNodeConnectedBehavior; defaultNodeConnectedBehavior = behavior; return prior == null; } - void clearBehaviors() { + public void clearBehaviors() { getConnectionBehaviors.clear(); nodeConnectedBehaviors.clear(); } - void clearBehavior(TransportAddress transportAddress) { + public void clearBehavior(TransportAddress transportAddress) { getConnectionBehaviors.remove(transportAddress); nodeConnectedBehaviors.remove(transportAddress); } From f429f9b6df1555b5571d434e6ba40d3773b2bdb5 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 27 Jul 2018 18:31:08 -0600 Subject: [PATCH 20/29] fix tests and temporarily mute test --- .../transport/TransportClientNodesServiceTests.java | 2 ++ .../elasticsearch/discovery/DiscoveryDisruptionIT.java | 3 +++ .../test/transport/MockTransportService.java | 9 --------- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 2b3374f8b7e46..cee0d2851306f 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.transport; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse; @@ -76,6 +77,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; +@LuceneTestCase.AwaitsFix(bugUrl = "") public class TransportClientNodesServiceTests extends ESTestCase { private static class TestIteration implements Closeable { diff --git a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java index 65ee9ff075066..4d190921f2da9 100644 --- a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.test.disruption.SlowClusterStateProcessing; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; @@ -177,6 +178,8 @@ public void testClusterJoinDespiteOfPublishingIssues() throws Exception { connection.sendRequest(requestId, action, request, options); }); + nonMasterTransportService.addConnectBehavior(masterTransportService, Transport::openConnection); + countDownLatch.await(); logger.info("waiting for cluster to reform"); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 86f94ebbde30b..bd8c66983f69c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -517,15 +517,6 @@ public boolean addNodeConnectedBehavior(NodeConnectedBehavior behavior) { return connectionManager().setDefaultNodeConnectedBehavior(behavior); } - /** - * Adds a new delegate transport that is used for communication with the given transport address. - * - * @return {@code true} if no other delegate was registered for this address before. - */ - public boolean addDelegate(TransportAddress transportAddress, DelegateTransport transport) { - return transport().transports.put(transportAddress, transport) == null; - } - public LookupTestTransport transport() { return (LookupTestTransport) transport; } From c45c5efb814f836759a7076f8cae00f244a708aa Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 31 Jul 2018 18:08:08 -0600 Subject: [PATCH 21/29] Fix tests --- .../transport/FailAndRetryMockTransport.java | 17 +++++++++++- .../TransportClientNodesServiceTests.java | 27 ++++++++++--------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java index 5dd2f8db506ee..8dfd8aecf508f 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java +++ b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.transport; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse; import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; @@ -29,6 +30,7 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; +import org.elasticsearch.common.concurrent.CompletableContext; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; @@ -42,6 +44,7 @@ import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseHandler; +import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportStats; import java.net.UnknownHostException; @@ -77,6 +80,8 @@ abstract class FailAndRetryMockTransport imp @Override public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { + CompletableContext closeContext = new CompletableContext<>(); + return new Connection() { @Override public DiscoveryNode getNode() { @@ -96,6 +101,10 @@ public void sendRequest(long requestId, String action, TransportRequest request, TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); ClusterState clusterState = getMockClusterState(node); transportResponseHandler.handleResponse(new ClusterStateResponse(clusterName, clusterState, 0L)); + } else if (TransportService.HANDSHAKE_ACTION_NAME.equals(action)) { + TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); + transportResponseHandler.handleResponse(new TransportService.HandshakeResponse(node, clusterName, node.getVersion())); + } else { throw new UnsupportedOperationException("Mock transport does not understand action " + action); } @@ -126,13 +135,19 @@ public void sendRequest(long requestId, String action, TransportRequest request, } } + @Override + public void addCloseListener(ActionListener listener) { + closeContext.addListener(ActionListener.toBiConsumer(listener)); + } + @Override public void close() { + closeContext.complete(null); } @Override public boolean isClosed() { - return false; + return closeContext.isDone(); } }; } diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index cee0d2851306f..91aa080df488b 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.client.transport; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse; @@ -65,6 +64,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import static org.elasticsearch.test.transport.MockTransportService.createNewService; import static org.hamcrest.CoreMatchers.equalTo; @@ -77,7 +77,6 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; -@LuceneTestCase.AwaitsFix(bugUrl = "") public class TransportClientNodesServiceTests extends ESTestCase { private static class TestIteration implements Closeable { @@ -388,18 +387,22 @@ public void testSniffNodesSamplerClosesConnections() throws Exception { transportClientNodesService.addTransportAddresses(remoteService.getLocalDiscoNode().getAddress()); assertEquals(1, transportClientNodesService.connectedNodes().size()); + assertTotalConnections(establishedConnections, 2); assertClosedConnections(establishedConnections, 1); transportClientNodesService.doSample(); + assertTotalConnections(establishedConnections, 3); + assertOpenConnections(establishedConnections, 1); assertClosedConnections(establishedConnections, 2); + assertTotalConnections(reusedConnections, 1); assertOpenConnections(reusedConnections, 1); handler.blockRequest(); Thread thread = new Thread(transportClientNodesService::doSample); thread.start(); - assertBusy(() -> assertEquals(3, establishedConnections.size())); - assertFalse("Temporary ping connection must be opened", establishedConnections.get(2).isClosed()); + assertBusy(() -> assertEquals(4, establishedConnections.size())); + assertFalse("Temporary ping connection must be opened", establishedConnections.get(3).isClosed()); handler.releaseRequest(); thread.join(); @@ -412,18 +415,18 @@ public void testSniffNodesSamplerClosesConnections() throws Exception { } } - private void assertClosedConnections(final List connections, final int size) { - assertEquals("Expecting " + size + " closed connections but got " + connections.size(), size, connections.size()); - connections.forEach(c -> assertConnection(c, true)); + private void assertTotalConnections(final List connections, final int size) { + assertEquals("Expecting " + size + " total connections but got " + connections.size(), size, connections.size()); } - private void assertOpenConnections(final List connections, final int size) { - assertEquals("Expecting " + size + " open connections but got " + connections.size(), size, connections.size()); - connections.forEach(c -> assertConnection(c, false)); + private void assertClosedConnections(final List connections, final int size) { + List closed = connections.stream().filter(MockConnection::isClosed).collect(Collectors.toList()); + assertEquals("Expecting " + size + " closed connections but got " + closed.size(), size, closed.size()); } - private static void assertConnection(final MockConnection connection, final boolean closed) { - assertEquals("Connection [" + connection + "] must be " + (closed ? "closed" : "open"), closed, connection.isClosed()); + private void assertOpenConnections(final List connections, final int size) { + List open = connections.stream().filter((c) -> c.isClosed() == false).collect(Collectors.toList()); + assertEquals("Expecting " + size + " open connections but got " + open.size(), size, open.size()); } class MockConnection implements Transport.Connection { From 2e8a0267b884ea7a242243683e2f020a4d4f8f39 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 31 Jul 2018 18:28:14 -0600 Subject: [PATCH 22/29] Fix checkstyle --- .../client/transport/FailAndRetryMockTransport.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java index 8dfd8aecf508f..9f45389ef6197 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java +++ b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.transport; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse; import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction; @@ -103,7 +104,8 @@ public void sendRequest(long requestId, String action, TransportRequest request, transportResponseHandler.handleResponse(new ClusterStateResponse(clusterName, clusterState, 0L)); } else if (TransportService.HANDSHAKE_ACTION_NAME.equals(action)) { TransportResponseHandler transportResponseHandler = responseHandlers.onResponseReceived(requestId, listener); - transportResponseHandler.handleResponse(new TransportService.HandshakeResponse(node, clusterName, node.getVersion())); + Version version = node.getVersion(); + transportResponseHandler.handleResponse(new TransportService.HandshakeResponse(node, clusterName, version)); } else { throw new UnsupportedOperationException("Mock transport does not understand action " + action); From 3ee4743f3ca00b287e7143ab449a444038d2f799 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 1 Aug 2018 13:13:35 -0600 Subject: [PATCH 23/29] Work on cleaning up --- .../elasticsearch/transport/Transport.java | 1 - .../indices/recovery/IndexRecoveryIT.java | 5 +- .../elasticsearch/recovery/RelocationIT.java | 3 +- .../test/transport/CapturingTransport.java | 2 +- .../test/transport/MockTransportService.java | 413 +++--------------- ...r.java => StubbableConnectionManager.java} | 72 +-- .../test/transport/StubbableTransport.java | 263 +++++++++++ 7 files changed, 347 insertions(+), 412 deletions(-) rename test/framework/src/main/java/org/elasticsearch/test/transport/{MockConnectionManager.java => StubbableConnectionManager.java} (52%) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java index ab278546da4b7..4ed593a9e6188 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transport.java +++ b/server/src/main/java/org/elasticsearch/transport/Transport.java @@ -134,7 +134,6 @@ default void addCloseListener(ActionListener listener) { default boolean isClosed() { - // TODO: should probably not be default return false; } diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 7e7f2981b4586..89a8813e3e07b 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -61,6 +61,7 @@ import org.elasticsearch.test.store.MockFSDirectoryService; import org.elasticsearch.test.store.MockFSIndexStore; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.test.transport.StubbableTransport; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportRequest; @@ -620,7 +621,7 @@ public void testDisconnectsWhileRecovering() throws Exception { } - private class RecoveryActionBlocker implements MockTransportService.SendRequestBehavior { + private class RecoveryActionBlocker implements StubbableTransport.SendRequestBehavior { private final boolean dropRequests; private final String recoveryActionToBlock; private final CountDownLatch requestBlocked; @@ -685,7 +686,7 @@ public void testDisconnectsDuringRecovery() throws Exception { MockTransportService blueMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, blueNodeName); MockTransportService redMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, redNodeName); - redMockTransportService.addSendBehavior(blueMockTransportService, new MockTransportService.SendRequestBehavior() { + redMockTransportService.addSendBehavior(blueMockTransportService, new StubbableTransport.SendRequestBehavior() { private final AtomicInteger count = new AtomicInteger(); @Override diff --git a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 6998bc81a4105..cb93d803bb7c6 100644 --- a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -58,6 +58,7 @@ import org.elasticsearch.test.MockIndexEventListener; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.test.transport.StubbableTransport; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; @@ -485,7 +486,7 @@ public void testIndexAndRelocateConcurrently() throws ExecutionException, Interr } - class RecoveryCorruption implements MockTransportService.SendRequestBehavior { + class RecoveryCorruption implements StubbableTransport.SendRequestBehavior { private final CountDownLatch corruptionCount; diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java index 69e5ffbd13397..b761a135c28d2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java @@ -93,7 +93,7 @@ public CapturedRequest(DiscoveryNode node, long requestId, String action, Transp public TransportService createCapturingTransportService(Settings settings, ThreadPool threadPool, TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, Set taskHeaders) { - MockConnectionManager connectionManager = new MockConnectionManager(new ConnectionManager(settings, this, threadPool), settings, + StubbableConnectionManager connectionManager = new StubbableConnectionManager(new ConnectionManager(settings, this, threadPool), settings, this, threadPool); connectionManager.setDefaultNodeConnectedBehavior((cm, discoveryNode) -> true); connectionManager.setDefaultConnectBehavior((cm, discoveryNode) -> new Connection() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index bd8c66983f69c..0326f1294169e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -26,8 +26,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.component.Lifecycle; -import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkService; @@ -39,7 +37,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; @@ -53,16 +50,12 @@ import org.elasticsearch.transport.RequestHandlerRegistry; import org.elasticsearch.transport.TcpTransport; import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportConnectionListener; -import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportInterceptor; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.transport.TransportStats; import java.io.IOException; -import java.net.UnknownHostException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -71,8 +64,6 @@ import java.util.Map; import java.util.Queue; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.atomic.AtomicBoolean; @@ -80,11 +71,11 @@ import java.util.function.Supplier; /** - * A mock transport service that allows to simulate different network topology failures. + * A mock delegate service that allows to simulate different network topology failures. * Internally it maps TransportAddress objects to rules that inject failures. * Adding rules for a node is done by adding rules for all bound addresses of a node * (and the publish address, if different). - * Matching requests to rules is based on the transport address associated with the + * Matching requests to rules is based on the delegate address associated with the * discovery node of the request, namely by DiscoveryNode.getAddress(). * This address is usually the publish address of the node but can also be a different one * (for example, @see org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing, which constructs @@ -149,15 +140,15 @@ public MockTransportService(Settings settings, Transport transport, ThreadPool t public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, Set taskHeaders) { - this(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); + this(settings, new StubbableTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); } - private MockTransportService(Settings settings, LookupTestTransport transport, ThreadPool threadPool, TransportInterceptor interceptor, + private MockTransportService(Settings settings, StubbableTransport transport, ThreadPool threadPool, TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, Set taskHeaders) { super(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders, - new MockConnectionManager(new ConnectionManager(settings, transport, threadPool), settings, transport, threadPool)); - this.original = transport.transport; + new StubbableConnectionManager(new ConnectionManager(settings, transport, threadPool), settings, transport, threadPool)); + this.original = transport.getDelegate(); } public static TransportAddress[] extractTransportAddresses(TransportService transportService) { @@ -183,11 +174,10 @@ protected TaskManager createTaskManager(Settings settings, ThreadPool threadPool public void clearAllRules() { transport().clearBehaviors(); connectionManager().clearBehaviors(); - transport().transports.clear(); } /** - * Clears the rule associated with the provided transport service. + * Clears the rule associated with the provided delegate service. */ public void clearRule(TransportService transportService) { for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { @@ -196,19 +186,11 @@ public void clearRule(TransportService transportService) { } /** - * Clears the rule associated with the provided transport address. + * Clears the rule associated with the provided delegate address. */ public void clearRule(TransportAddress transportAddress) { transport().clearBehavior(transportAddress); connectionManager().clearBehavior(transportAddress); - transport().transports.remove(transportAddress); - } - - /** - * Returns the original Transport service wrapped by this mock transport service. - */ - public Transport original() { - return original; } /** @@ -341,7 +323,7 @@ public void addUnresponsiveRule(TransportAddress transportAddress, final TimeVal } }); - transport().addSendBehavior(transportAddress, new SendRequestBehavior() { + transport().addSendBehavior(transportAddress, new StubbableTransport.SendRequestBehavior() { private final Queue requestsToSendWhenCleared = new LinkedBlockingDeque<>(); private boolean cleared = false; @@ -400,11 +382,11 @@ public void clearCallback() { } /** - * Adds a new send behavior that is used for communication with the given transport service. + * Adds a new send behavior that is used for communication with the given delegate service. * - * @return {@code true} if no other send behavior was registered for any of the addresses bound by transport service. + * @return {@code true} if no other send behavior was registered for any of the addresses bound by delegate service. */ - public boolean addSendBehavior(TransportService transportService, SendRequestBehavior sendBehavior) { + public boolean addSendBehavior(TransportService transportService, StubbableTransport.SendRequestBehavior sendBehavior) { boolean noRegistered = true; for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { noRegistered &= addSendBehavior(transportAddress, sendBehavior); @@ -413,11 +395,11 @@ public boolean addSendBehavior(TransportService transportService, SendRequestBeh } /** - * Adds a new send behavior that is used for communication with the given transport address. + * Adds a new send behavior that is used for communication with the given delegate address. * * @return {@code true} if no other send behavior was registered for this address before. */ - public boolean addSendBehavior(TransportAddress transportAddress, SendRequestBehavior sendBehavior) { + public boolean addSendBehavior(TransportAddress transportAddress, StubbableTransport.SendRequestBehavior sendBehavior) { return transport().addSendBehavior(transportAddress, sendBehavior); } @@ -426,19 +408,17 @@ public boolean addSendBehavior(TransportAddress transportAddress, SendRequestBeh * * @return {@code true} if no default send behavior was registered */ - public boolean addSendBehavior(SendRequestBehavior behavior) { - SendRequestBehavior prior = transport().defaultSendRequest; - transport().defaultSendRequest = behavior; - return prior == null; + public boolean addSendBehavior(StubbableTransport.SendRequestBehavior behavior) { + return transport().setDefaultSendBehavior(behavior); } /** - * Adds a new connect behavior that is used for creating connections with the given transport service. + * Adds a new connect behavior that is used for creating connections with the given delegate service. * - * @return {@code true} if no other send behavior was registered for any of the addresses bound by transport service. + * @return {@code true} if no other send behavior was registered for any of the addresses bound by delegate service. */ - public boolean addConnectBehavior(TransportService transportService, ConnectBehavior connectBehavior) { + public boolean addConnectBehavior(TransportService transportService, StubbableTransport.OpenConnectionBehavior connectBehavior) { boolean noRegistered = true; for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { noRegistered &= addConnectBehavior(transportAddress, connectBehavior); @@ -447,20 +427,20 @@ public boolean addConnectBehavior(TransportService transportService, ConnectBeha } /** - * Adds a new connect behavior that is used for creating connections with the given transport address. + * Adds a new connect behavior that is used for creating connections with the given delegate address. * * @return {@code true} if no other send behavior was registered for this address before. */ - public boolean addConnectBehavior(TransportAddress transportAddress, ConnectBehavior connectBehavior) { + public boolean addConnectBehavior(TransportAddress transportAddress, StubbableTransport.OpenConnectionBehavior connectBehavior) { return transport().addConnectBehavior(transportAddress, connectBehavior); } /** - * Adds a new get connection behavior that is used for communication with the given transport service. + * Adds a new get connection behavior that is used for communication with the given delegate service. * - * @return {@code true} if no other get connection behavior was registered for any of the addresses bound by transport service. + * @return {@code true} if no other get connection behavior was registered for any of the addresses bound by delegate service. */ - public boolean addGetConnectionBehavior(TransportService transportService, GetConnectionBehavior behavior) { + public boolean addGetConnectionBehavior(TransportService transportService, StubbableConnectionManager.GetConnectionBehavior behavior) { boolean noRegistered = true; for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { noRegistered &= addGetConnectionBehavior(transportAddress, behavior); @@ -469,11 +449,11 @@ public boolean addGetConnectionBehavior(TransportService transportService, GetCo } /** - * Adds a get connection behavior that is used for communication with the given transport address. + * Adds a get connection behavior that is used for communication with the given delegate address. * * @return {@code true} if no other get connection behavior was registered for this address before. */ - public boolean addGetConnectionBehavior(TransportAddress transportAddress, GetConnectionBehavior behavior) { + public boolean addGetConnectionBehavior(TransportAddress transportAddress, StubbableConnectionManager.GetConnectionBehavior behavior) { return connectionManager().addConnectBehavior(transportAddress, behavior); } @@ -482,16 +462,16 @@ public boolean addGetConnectionBehavior(TransportAddress transportAddress, GetCo * * @return {@code true} if no default get connection behavior was registered. */ - public boolean addGetConnectionBehavior(GetConnectionBehavior behavior) { + public boolean addGetConnectionBehavior(StubbableConnectionManager.GetConnectionBehavior behavior) { return connectionManager().setDefaultConnectBehavior(behavior); } /** - * Adds a node connected behavior that is used for the given transport service. + * Adds a node connected behavior that is used for the given delegate service. * - * @return {@code true} if no other node connected behavior was registered for any of the addresses bound by transport service. + * @return {@code true} if no other node connected behavior was registered for any of the addresses bound by delegate service. */ - public boolean addNodeConnectedBehavior(TransportService transportService, NodeConnectedBehavior behavior) { + public boolean addNodeConnectedBehavior(TransportService transportService, StubbableConnectionManager.NodeConnectedBehavior behavior) { boolean noRegistered = true; for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { noRegistered &= addNodeConnectedBehavior(transportAddress, behavior); @@ -500,11 +480,11 @@ public boolean addNodeConnectedBehavior(TransportService transportService, NodeC } /** - * Adds a node connected behavior that is used for the given transport address. + * Adds a node connected behavior that is used for the given delegate address. * * @return {@code true} if no other node connected behavior was registered for this address before. */ - public boolean addNodeConnectedBehavior(TransportAddress transportAddress, NodeConnectedBehavior behavior) { + public boolean addNodeConnectedBehavior(TransportAddress transportAddress, StubbableConnectionManager.NodeConnectedBehavior behavior) { return connectionManager().addNodeConnectedBehavior(transportAddress, behavior); } @@ -513,235 +493,16 @@ public boolean addNodeConnectedBehavior(TransportAddress transportAddress, NodeC * * @return {@code true} if no default node connected behavior was registered. */ - public boolean addNodeConnectedBehavior(NodeConnectedBehavior behavior) { + public boolean addNodeConnectedBehavior(StubbableConnectionManager.NodeConnectedBehavior behavior) { return connectionManager().setDefaultNodeConnectedBehavior(behavior); } - public LookupTestTransport transport() { - return (LookupTestTransport) transport; - } - - public MockConnectionManager connectionManager() { - return (MockConnectionManager) connectionManager; + public StubbableTransport transport() { + return (StubbableTransport) transport; } - /** - * A lookup transport that has a list of potential Transport implementations to delegate to for node operations, - * if none is registered, then the default one is used. - */ - private static class LookupTestTransport extends DelegateTransport { - - private final ConcurrentMap transports = ConcurrentCollections.newConcurrentMap(); - private final ConcurrentHashMap sendBehaviors = new ConcurrentHashMap<>(); - private final ConcurrentHashMap connectBehaviors = new ConcurrentHashMap<>(); - private volatile MockTransportService.SendRequestBehavior defaultSendRequest = null; - private volatile MockTransportService.ConnectBehavior defaultConnectBehavior = null; - - private LookupTestTransport(Transport transport) { - super(transport); - } - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - TransportAddress address = node.getAddress(); - MockTransportService.ConnectBehavior behavior = connectBehaviors.getOrDefault(address, defaultConnectBehavior); - Connection connection; - if (behavior == null) { - connection = transport.openConnection(node, profile); - } else { - connection = behavior.openConnection(transport, node, profile); - } - - return new WrappedConnection(connection); - } - - boolean addSendBehavior(TransportAddress transportAddress, MockTransportService.SendRequestBehavior sendBehavior) { - return sendBehaviors.put(transportAddress, sendBehavior) == null; - } - - boolean addConnectBehavior(TransportAddress transportAddress, MockTransportService.ConnectBehavior connectBehavior) { - return connectBehaviors.put(transportAddress, connectBehavior) == null; - } - - void clearBehaviors() { - sendBehaviors.clear(); - connectBehaviors.clear(); - } - - void clearBehavior(TransportAddress transportAddress) { - MockTransportService.SendRequestBehavior behavior = sendBehaviors.remove(transportAddress); - if (behavior != null) { - behavior.clearCallback(); - } - connectBehaviors.remove(transportAddress); - } - - private class WrappedConnection implements Transport.Connection { - - private final Transport.Connection connection; - - private WrappedConnection(Transport.Connection connection) { - this.connection = connection; - } - - @Override - public DiscoveryNode getNode() { - return connection.getNode(); - } - - @Override - public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws IOException, TransportException { - TransportAddress address = connection.getNode().getAddress(); - MockTransportService.SendRequestBehavior behavior = sendBehaviors.getOrDefault(address, defaultSendRequest); - if (behavior == null) { - connection.sendRequest(requestId, action, request, options); - } else { - // TODO: Connections maybe not the same? - behavior.sendRequest(connection, requestId, action, request, options); - } - } - - @Override - public boolean sendPing() { - return connection.sendPing(); - } - - @Override - public void addCloseListener(ActionListener listener) { - connection.addCloseListener(listener); - } - - - @Override - public boolean isClosed() { - return connection.isClosed(); - } - - @Override - public Version getVersion() { - return connection.getVersion(); - } - - @Override - public Object getCacheKey() { - return connection.getCacheKey(); - } - - @Override - public void close() throws IOException { - connection.close(); - } - - @Override - public boolean equals(Object obj) { - return connection.equals(obj); - } - - @Override - public int hashCode() { - return connection.hashCode(); - } - } - } - - /** - * A pure delegate transport. - * Can be extracted to a common class if needed in other places in the codebase. - */ - public static class DelegateTransport implements Transport { - - protected final Transport transport; - - - public DelegateTransport(Transport transport) { - this.transport = transport; - } - - @Override - public void addConnectionListener(TransportConnectionListener listener) { - transport.addConnectionListener(listener); - } - - @Override - public boolean removeConnectionListener(TransportConnectionListener listener) { - return transport.removeConnectionListener(listener); - } - - @Override - public void registerRequestHandler(RequestHandlerRegistry reg) { - transport.registerRequestHandler(reg); - } - - @Override - public RequestHandlerRegistry getRequestHandler(String action) { - return transport.getRequestHandler(action); - } - - @Override - public BoundTransportAddress boundAddress() { - return transport.boundAddress(); - } - - @Override - public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException { - return transport.addressesFromString(address, perAddressLimit); - } - - @Override - public List getLocalAddresses() { - return transport.getLocalAddresses(); - } - - @Override - public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - return transport.openConnection(node, profile); - } - - @Override - public TransportStats getStats() { - return transport.getStats(); - } - - @Override - public ResponseHandlers getResponseHandlers() { - return transport.getResponseHandlers(); - } - - @Override - public Lifecycle.State lifecycleState() { - return transport.lifecycleState(); - } - - @Override - public void addLifecycleListener(LifecycleListener listener) { - transport.addLifecycleListener(listener); - } - - @Override - public void removeLifecycleListener(LifecycleListener listener) { - transport.removeLifecycleListener(listener); - } - - @Override - public void start() { - transport.start(); - } - - @Override - public void stop() { - transport.stop(); - } - - @Override - public void close() { - transport.close(); - } - - @Override - public Map profileBoundAddresses() { - return transport.profileBoundAddresses(); - } + public StubbableConnectionManager connectionManager() { + return (StubbableConnectionManager) connectionManager; } List activeTracers = new CopyOnWriteArrayList<>(); @@ -820,78 +581,36 @@ protected void traceRequestSent(DiscoveryNode node, long requestId, String actio } } - private static class FilteredConnection implements Transport.Connection { - protected final Transport.Connection connection; - - private FilteredConnection(Transport.Connection connection) { - this.connection = connection; - } - - @Override - public DiscoveryNode getNode() { - return connection.getNode(); - } - - @Override - public Version getVersion() { - return connection.getVersion(); - } - - @Override - public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws IOException, TransportException { - connection.sendRequest(requestId, action, request, options); - } - - @Override - public void close() throws IOException { - connection.close(); - } - - @Override - public Object getCacheKey() { - return connection.getCacheKey(); - } - } - public Transport getOriginalTransport() { - Transport transport = transport(); - while (transport instanceof DelegateTransport) { - transport = ((DelegateTransport) transport).transport; + StubbableTransport transport = transport(); + while (transport instanceof StubbableTransport) { + transport = (StubbableTransport) transport.getDelegate(); } return transport; } @Override public Transport.Connection openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException { - FilteredConnection filteredConnection = new FilteredConnection(super.openConnection(node, profile)) { - final AtomicBoolean closed = new AtomicBoolean(false); - - @Override - public void close() throws IOException { - try { - super.close(); - } finally { - if (closed.compareAndSet(false, true)) { - synchronized (openConnections) { - List connections = openConnections.get(node); - boolean remove = connections.remove(this); - assert remove; - if (connections.isEmpty()) { - openConnections.remove(node); - } - } - } - } + Transport.Connection connection = super.openConnection(node, profile); - } - }; synchronized (openConnections) { List connections = openConnections.computeIfAbsent(node, (n) -> new CopyOnWriteArrayList<>()); - connections.add(filteredConnection); + connections.add(connection); } - return filteredConnection; + + connection.addCloseListener(ActionListener.wrap(() -> { + synchronized (openConnections) { + List connections = openConnections.get(node); + boolean remove = connections.remove(connection); + assert remove; + if (connections.isEmpty()) { + openConnections.remove(node); + } + } + })); + + return connection; } @Override @@ -906,26 +625,4 @@ public DiscoveryNode getLocalDiscoNode() { return this.getLocalNode(); } - @FunctionalInterface - public interface SendRequestBehavior { - void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException; - - default void clearCallback() {} - } - - @FunctionalInterface - public interface ConnectBehavior { - Transport.Connection openConnection(Transport transport, DiscoveryNode discoveryNode, ConnectionProfile profile); - } - - @FunctionalInterface - public interface GetConnectionBehavior { - Transport.Connection getConnection(ConnectionManager connectionManager, DiscoveryNode discoveryNode); - } - - @FunctionalInterface - public interface NodeConnectedBehavior { - boolean nodeConnected(ConnectionManager connectionManager, DiscoveryNode discoveryNode); - } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java similarity index 52% rename from test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java rename to test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java index 099919af20517..ec92dfe43789f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockConnectionManager.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java @@ -33,37 +33,37 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -public class MockConnectionManager extends ConnectionManager { +public class StubbableConnectionManager extends ConnectionManager { private final ConnectionManager delegate; - private final ConcurrentMap getConnectionBehaviors; - private final ConcurrentMap nodeConnectedBehaviors; - private volatile MockTransportService.GetConnectionBehavior defaultGetConnectionBehavior = null; - private volatile MockTransportService.NodeConnectedBehavior defaultNodeConnectedBehavior = null; + private final ConcurrentMap getConnectionBehaviors; + private final ConcurrentMap nodeConnectedBehaviors; + private volatile GetConnectionBehavior defaultGetConnectionBehavior = ConnectionManager::getConnection; + private volatile NodeConnectedBehavior defaultNodeConnectedBehavior = ConnectionManager::nodeConnected; - public MockConnectionManager(ConnectionManager delegate, Settings settings, Transport transport, ThreadPool threadPool) { + public StubbableConnectionManager(ConnectionManager delegate, Settings settings, Transport transport, ThreadPool threadPool) { super(settings, transport, threadPool); this.delegate = delegate; this.getConnectionBehaviors = new ConcurrentHashMap<>(); this.nodeConnectedBehaviors = new ConcurrentHashMap<>(); } - public boolean addConnectBehavior(TransportAddress transportAddress, MockTransportService.GetConnectionBehavior connectBehavior) { + public boolean addConnectBehavior(TransportAddress transportAddress, GetConnectionBehavior connectBehavior) { return getConnectionBehaviors.put(transportAddress, connectBehavior) == null; } - public boolean setDefaultConnectBehavior(MockTransportService.GetConnectionBehavior behavior) { - MockTransportService.GetConnectionBehavior prior = defaultGetConnectionBehavior; + public boolean setDefaultConnectBehavior(GetConnectionBehavior behavior) { + GetConnectionBehavior prior = defaultGetConnectionBehavior; defaultGetConnectionBehavior = behavior; return prior == null; } - public boolean addNodeConnectedBehavior(TransportAddress transportAddress, MockTransportService.NodeConnectedBehavior behavior) { + public boolean addNodeConnectedBehavior(TransportAddress transportAddress, NodeConnectedBehavior behavior) { return nodeConnectedBehaviors.put(transportAddress, behavior) == null; } - public boolean setDefaultNodeConnectedBehavior(MockTransportService.NodeConnectedBehavior behavior) { - MockTransportService.NodeConnectedBehavior prior = defaultNodeConnectedBehavior; + public boolean setDefaultNodeConnectedBehavior(NodeConnectedBehavior behavior) { + NodeConnectedBehavior prior = defaultNodeConnectedBehavior; defaultNodeConnectedBehavior = behavior; return prior == null; } @@ -81,50 +81,24 @@ public void clearBehavior(TransportAddress transportAddress) { @Override public Transport.Connection getConnection(DiscoveryNode node) { TransportAddress address = node.getAddress(); - MockTransportService.GetConnectionBehavior behavior = getConnectionBehaviors.getOrDefault(address, defaultGetConnectionBehavior); - if (behavior == null) { - return delegate.getConnection(node); - } else { - return behavior.getConnection(delegate, node); - } + GetConnectionBehavior behavior = getConnectionBehaviors.getOrDefault(address, defaultGetConnectionBehavior); + return behavior.getConnection(delegate, node); } + @Override public boolean nodeConnected(DiscoveryNode node) { TransportAddress address = node.getAddress(); - MockTransportService.NodeConnectedBehavior behavior = nodeConnectedBehaviors.getOrDefault(address, defaultNodeConnectedBehavior); - if (behavior == null) { - return delegate.nodeConnected(node); - } else { - return behavior.nodeConnected(delegate, node); - } - } - - @Override - public void addListener(TransportConnectionListener listener) { - delegate.addListener(listener); - } - - @Override - public void removeListener(TransportConnectionListener listener) { - delegate.removeListener(listener); - } - - public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, - CheckedBiConsumer connectionValidator) - throws ConnectTransportException { - delegate.connectToNode(node, connectionProfile, connectionValidator); + NodeConnectedBehavior behavior = nodeConnectedBehaviors.getOrDefault(address, defaultNodeConnectedBehavior); + return behavior.nodeConnected(delegate, node); } - public void disconnectFromNode(DiscoveryNode node) { - delegate.disconnectFromNode(node); + @FunctionalInterface + public interface GetConnectionBehavior { + Transport.Connection getConnection(ConnectionManager connectionManager, DiscoveryNode discoveryNode); } - public int connectedNodeCount() { - return delegate.connectedNodeCount(); - } - - @Override - public void close() { - delegate.close(); + @FunctionalInterface + public interface NodeConnectedBehavior { + boolean nodeConnected(ConnectionManager connectionManager, DiscoveryNode discoveryNode); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java new file mode 100644 index 0000000000000..fcd997202aa7a --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java @@ -0,0 +1,263 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.transport; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.component.LifecycleListener; +import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.transport.ConnectionProfile; +import org.elasticsearch.transport.RequestHandlerRegistry; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportConnectionListener; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportStats; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class StubbableTransport implements Transport { + + private final ConcurrentHashMap sendBehaviors = new ConcurrentHashMap<>(); + private final ConcurrentHashMap connectBehaviors = new ConcurrentHashMap<>(); + private volatile SendRequestBehavior defaultSendRequest = null; + private volatile OpenConnectionBehavior defaultConnectBehavior = null; + private final Transport delegate; + + + public StubbableTransport(Transport transport) { + this.delegate = transport; + } + + boolean setDefaultSendBehavior(SendRequestBehavior sendBehavior) { + SendRequestBehavior prior = defaultSendRequest; + defaultSendRequest = sendBehavior; + return prior == null; + } + + boolean addSendBehavior(TransportAddress transportAddress, SendRequestBehavior sendBehavior) { + return sendBehaviors.put(transportAddress, sendBehavior) == null; + } + + boolean addConnectBehavior(TransportAddress transportAddress, OpenConnectionBehavior connectBehavior) { + return connectBehaviors.put(transportAddress, connectBehavior) == null; + } + + void clearBehaviors() { + sendBehaviors.clear(); + connectBehaviors.clear(); + } + + void clearBehavior(TransportAddress transportAddress) { + SendRequestBehavior behavior = sendBehaviors.remove(transportAddress); + if (behavior != null) { + behavior.clearCallback(); + } + connectBehaviors.remove(transportAddress); + } + + Transport getDelegate() { + return delegate; + } + + @Override + public void addConnectionListener(TransportConnectionListener listener) { + delegate.addConnectionListener(listener); + } + + @Override + public boolean removeConnectionListener(TransportConnectionListener listener) { + return delegate.removeConnectionListener(listener); + } + + @Override + public void registerRequestHandler(RequestHandlerRegistry reg) { + delegate.registerRequestHandler(reg); + } + + @Override + public RequestHandlerRegistry getRequestHandler(String action) { + return delegate.getRequestHandler(action); + } + + @Override + public BoundTransportAddress boundAddress() { + return delegate.boundAddress(); + } + + @Override + public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws UnknownHostException { + return delegate.addressesFromString(address, perAddressLimit); + } + + @Override + public List getLocalAddresses() { + return delegate.getLocalAddresses(); + } + + @Override + public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { + TransportAddress address = node.getAddress(); + OpenConnectionBehavior behavior = connectBehaviors.getOrDefault(address, defaultConnectBehavior); + Connection connection; + if (behavior == null) { + connection = delegate.openConnection(node, profile); + } else { + connection = behavior.openConnection(delegate, node, profile); + } + + return new WrappedConnection(connection); + } + + @Override + public TransportStats getStats() { + return delegate.getStats(); + } + + @Override + public Transport.ResponseHandlers getResponseHandlers() { + return delegate.getResponseHandlers(); + } + + @Override + public Lifecycle.State lifecycleState() { + return delegate.lifecycleState(); + } + + @Override + public void addLifecycleListener(LifecycleListener listener) { + delegate.addLifecycleListener(listener); + } + + @Override + public void removeLifecycleListener(LifecycleListener listener) { + delegate.removeLifecycleListener(listener); + } + + @Override + public void start() { + delegate.start(); + } + + @Override + public void stop() { + delegate.stop(); + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public Map profileBoundAddresses() { + return delegate.profileBoundAddresses(); + } + + private class WrappedConnection implements Transport.Connection { + + + private final Transport.Connection connection; + + private WrappedConnection(Transport.Connection connection) { + this.connection = connection; + } + + @Override + public DiscoveryNode getNode() { + return connection.getNode(); + } + + @Override + public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) + throws IOException, TransportException { + TransportAddress address = connection.getNode().getAddress(); + SendRequestBehavior behavior = sendBehaviors.getOrDefault(address, defaultSendRequest); + if (behavior == null) { + connection.sendRequest(requestId, action, request, options); + } else { + behavior.sendRequest(connection, requestId, action, request, options); + } + } + + @Override + public boolean sendPing() { + return connection.sendPing(); + } + + @Override + public void addCloseListener(ActionListener listener) { + connection.addCloseListener(listener); + } + + + @Override + public boolean isClosed() { + return connection.isClosed(); + } + + @Override + public Version getVersion() { + return connection.getVersion(); + } + + @Override + public Object getCacheKey() { + return connection.getCacheKey(); + } + + @Override + public void close() throws IOException { + connection.close(); + } + + @Override + public boolean equals(Object obj) { + return connection.equals(obj); + } + + @Override + public int hashCode() { + return connection.hashCode(); + } + } + + @FunctionalInterface + public interface OpenConnectionBehavior { + Connection openConnection(Transport transport, DiscoveryNode discoveryNode, ConnectionProfile profile); + } + + @FunctionalInterface + public interface SendRequestBehavior { + void sendRequest(Connection connection, long requestId, String action, TransportRequest request, + TransportRequestOptions options) throws IOException; + + default void clearCallback() { + } + } +} From 2789d0d7d31eb02aacb222fb36ee6e61a03604fd Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 1 Aug 2018 14:30:28 -0600 Subject: [PATCH 24/29] Make connection methods required --- .../elasticsearch/transport/Transport.java | 10 +++----- .../transport/TransportService.java | 10 ++++++++ .../action/search/SearchAsyncActionTests.java | 10 ++++++++ .../TransportClientNodesServiceTests.java | 7 ++---- .../cluster/NodeConnectionsServiceTests.java | 6 +++++ .../RemoteClusterConnectionTests.java | 20 +++++++++++++++ .../test/transport/CapturingTransport.java | 25 +++++++++++++++++-- .../transport/StubbableConnectionManager.java | 5 ---- 8 files changed, 74 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java index 4ed593a9e6188..99d192d9201a0 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transport.java +++ b/server/src/main/java/org/elasticsearch/transport/Transport.java @@ -128,14 +128,9 @@ default boolean sendPing() { return false; } - default void addCloseListener(ActionListener listener) { - throw new UnsupportedOperationException("Not supported"); - } + void addCloseListener(ActionListener listener); - - default boolean isClosed() { - return false; - } + boolean isClosed(); /** * Returns the version of the node this connection was established with. @@ -151,6 +146,7 @@ default Version getVersion() { default Object getCacheKey() { return this; } + } /** diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 48c743604e8e4..d7ece36d7fdda 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; @@ -133,6 +134,15 @@ public void sendRequest(long requestId, String action, TransportRequest request, sendLocalRequest(requestId, action, request, options); } + @Override + public void addCloseListener(ActionListener listener) { + } + + @Override + public boolean isClosed() { + return false; + } + @Override public void close() { } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index 2726de237582d..95525ac8c2ba7 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -443,6 +443,16 @@ public void sendRequest(long requestId, String action, TransportRequest request, throw new UnsupportedOperationException(); } + @Override + public void addCloseListener(ActionListener listener) { + + } + + @Override + public boolean isClosed() { + return false; + } + @Override public void close() throws IOException { throw new UnsupportedOperationException(); diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 91aa080df488b..8c18cabf7f3d4 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -430,7 +430,6 @@ private void assertOpenConnections(final List connections, final } class MockConnection implements Transport.Connection { - private final AtomicBoolean closed = new AtomicBoolean(false); private final Transport.Connection connection; private MockConnection(Transport.Connection connection) { @@ -455,9 +454,7 @@ public void sendRequest(long requestId, String action, TransportRequest request, @Override public void close() throws IOException { - if (closed.compareAndSet(false, true)) { - connection.close(); - } + connection.close(); } @Override @@ -472,7 +469,7 @@ public void addCloseListener(ActionListener listener) { @Override public boolean isClosed() { - return closed.get(); + return connection.isClosed(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index 6f138aadd08e2..92c1016d3e615 100644 --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.UUIDs; @@ -244,6 +245,11 @@ public void sendRequest(long requestId, String action, TransportRequest request, } + @Override + public void addCloseListener(ActionListener listener) { + + } + @Override public void close() { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 4de4314979ca2..9df80ae9e5a20 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -435,6 +435,16 @@ public void sendRequest(long requestId, String action, TransportRequest request, // no-op } + @Override + public void addCloseListener(ActionListener listener) { + // no-op + } + + @Override + public boolean isClosed() { + return false; + } + @Override public void close() throws IOException { // no-op @@ -1287,6 +1297,16 @@ public void sendRequest(long requestId, String action, TransportRequest request, // no-op } + @Override + public void addCloseListener(ActionListener listener) { + // no-op + } + + @Override + public boolean isClosed() { + return false; + } + @Override public void close() { // no-op diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java index b761a135c28d2..a7d3e85d3009d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java @@ -20,6 +20,7 @@ package org.elasticsearch.test.transport; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Randomness; @@ -93,8 +94,8 @@ public CapturedRequest(DiscoveryNode node, long requestId, String action, Transp public TransportService createCapturingTransportService(Settings settings, ThreadPool threadPool, TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, Set taskHeaders) { - StubbableConnectionManager connectionManager = new StubbableConnectionManager(new ConnectionManager(settings, this, threadPool), settings, - this, threadPool); + StubbableConnectionManager connectionManager = new StubbableConnectionManager(new ConnectionManager(settings, this, threadPool), + settings, this, threadPool); connectionManager.setDefaultNodeConnectedBehavior((cm, discoveryNode) -> true); connectionManager.setDefaultConnectBehavior((cm, discoveryNode) -> new Connection() { @Override @@ -109,6 +110,16 @@ public void sendRequest(long requestId, String action, TransportRequest request, capturedRequests.add(new CapturedRequest(discoveryNode, requestId, action, request)); } + @Override + public void addCloseListener(ActionListener listener) { + + } + + @Override + public boolean isClosed() { + return false; + } + @Override public void close() { @@ -244,6 +255,16 @@ public void sendRequest(long requestId, String action, TransportRequest request, capturedRequests.add(new CapturedRequest(node, requestId, action, request)); } + @Override + public void addCloseListener(ActionListener listener) { + + } + + @Override + public boolean isClosed() { + return false; + } + @Override public void close() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java index ec92dfe43789f..b3882801c5fc8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java @@ -19,17 +19,12 @@ package org.elasticsearch.test.transport; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.ConnectionManager; -import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportConnectionListener; -import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; From 617521fdea42007e133c73e22f19136bceb57752 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 1 Aug 2018 17:07:42 -0600 Subject: [PATCH 25/29] Work on tests --- .../transport/CloseableConnection.java | 47 +++++ .../transport/ConnectionManager.java | 129 ++++++++------ .../elasticsearch/transport/TcpTransport.java | 39 ++--- .../elasticsearch/transport/Transport.java | 2 + .../action/search/SearchAsyncActionTests.java | 2 +- .../TransportClientNodesServiceTests.java | 2 +- .../transport/ConnectionManagerTests.java | 164 ++++++++++++++++++ .../RemoteClusterConnectionTests.java | 2 +- .../test/transport/StubbableTransport.java | 2 +- 9 files changed, 309 insertions(+), 80 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/transport/CloseableConnection.java create mode 100644 server/src/test/java/org/elasticsearch/transport/ConnectionManagerTests.java diff --git a/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java b/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java new file mode 100644 index 0000000000000..3932fdb43485c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.concurrent.CompletableContext; + + +/** + * Abstract Transport.Connection that provides common close logic. + */ +public abstract class CloseableConnection implements Transport.Connection { + + private final CompletableContext closeContext = new CompletableContext<>(); + + @Override + public void addCloseListener(ActionListener listener) { + closeContext.addListener(ActionListener.toBiConsumer(listener)); + } + + @Override + public boolean isClosed() { + return closeContext.isDone(); + } + + @Override + public void close() { + closeContext.complete(null); + } +} diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index 7906efaabae14..744d979be1fec 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.KeyedLock; import org.elasticsearch.core.internal.io.IOUtils; @@ -41,18 +42,21 @@ import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; - -import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class ConnectionManager implements Closeable { - private final ConcurrentMap connectedNodes = newConcurrentMap(); + private final ConcurrentMap connectedNodes = ConcurrentCollections.newConcurrentMap(); private final KeyedLock connectionLock = new KeyedLock<>(); private final Logger logger; private final Transport transport; private final ThreadPool threadPool; private final TimeValue pingSchedule; private final Lifecycle lifecycle = new Lifecycle(); + private final ReadWriteLock closeLock = new ReentrantReadWriteLock(); private final DelegatingNodeConnectionListener connectionListener = new DelegatingNodeConnectionListener(); public ConnectionManager(Settings settings, Transport transport, ThreadPool threadPool) { @@ -81,45 +85,49 @@ public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfil if (node == null) { throw new ConnectTransportException(null, "can't connect to a null node"); } - ensureOpen(); - try (Releasable ignored = connectionLock.acquire(node.getId())) { - Transport.Connection connection = connectedNodes.get(node); - if (connection != null) { - return; - } - boolean success = false; - try { - connection = transport.openConnection(node, connectionProfile); - connectionValidator.accept(connection, connectionProfile); - // we acquire a connection lock, so no way there is an existing connection - connectedNodes.put(node, connection); - if (logger.isDebugEnabled()) { - logger.debug("connected to node [{}]", node); + closeLock.readLock().lock(); // ensure we don't open connections while we are closing + try { + ensureOpen(); + try (Releasable ignored = connectionLock.acquire(node.getId())) { + Transport.Connection connection = connectedNodes.get(node); + if (connection != null) { + return; } + boolean success = false; try { - connectionListener.onNodeConnected(node); - } finally { - final Transport.Connection finalConnection = connection; - connection.addCloseListener(ActionListener.wrap(() -> { - if (connectedNodes.remove(node, finalConnection)) { + connection = transport.openConnection(node, connectionProfile); + connectionValidator.accept(connection, connectionProfile); + // we acquire a connection lock, so no way there is an existing connection + connectedNodes.put(node, connection); + if (logger.isDebugEnabled()) { + logger.debug("connected to node [{}]", node); + } + try { + connectionListener.onNodeConnected(node); + } finally { + final Transport.Connection finalConnection = connection; + connection.addCloseListener(ActionListener.wrap(() -> { + connectedNodes.remove(node, finalConnection); connectionListener.onNodeDisconnected(node); - } - })); - } - if (connection.isClosed()) { - throw new NodeNotConnectedException(node, "connection concurrently closed"); - } - success = true; - } catch (ConnectTransportException e) { - throw e; - } catch (Exception e) { - throw new ConnectTransportException(node, "general node connection failure", e); - } finally { - if (success == false) { // close the connection if there is a failure - logger.trace(() -> new ParameterizedMessage("failed to connect to [{}], cleaning dangling connections", node)); - IOUtils.closeWhileHandlingException(connection); + })); + } + if (connection.isClosed()) { + throw new NodeNotConnectedException(node, "connection concurrently closed"); + } + success = true; + } catch (ConnectTransportException e) { + throw e; + } catch (Exception e) { + throw new ConnectTransportException(node, "general node connection failure", e); + } finally { + if (success == false) { // close the connection if there is a failure + logger.trace(() -> new ParameterizedMessage("failed to connect to [{}], cleaning dangling connections", node)); + IOUtils.closeWhileHandlingException(connection); + } } } + } finally { + closeLock.readLock().unlock(); } } @@ -136,10 +144,10 @@ public boolean nodeConnected(DiscoveryNode node) { } public void disconnectFromNode(DiscoveryNode node) { - // TODO: Do we need to lock here? Transport.Connection nodeChannels = connectedNodes.remove(node); - if (nodeChannels != null) { // if we found it and removed it we close and notify - IOUtils.closeWhileHandlingException(nodeChannels, () -> connectionListener.onNodeDisconnected(node)); + if (nodeChannels != null) { + // if we found it and removed it we close + nodeChannels.close(); } } @@ -150,21 +158,40 @@ public int connectedNodeCount() { @Override public void close() { lifecycle.moveToStopped(); - // TODO: Either add locking externally or in here. - // we are holding a write lock so nobody modifies the connectedNodes / openConnections map - it's safe to first close - // all instances and then clear them maps - Iterator> iterator = connectedNodes.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry next = iterator.next(); + CountDownLatch latch = new CountDownLatch(1); + + // TODO: Consider moving all read/write lock (in Transport and this class) to the TransportService + threadPool.generic().execute(() -> { + closeLock.writeLock().lock(); try { - IOUtils.closeWhileHandlingException(next.getValue()); - connectionListener.onNodeDisconnected(next.getKey()); + // we are holding a write lock so nobody modifies the connectedNodes / openConnections map - it's safe to first close + // all instances and then clear them maps + Iterator> iterator = connectedNodes.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry next = iterator.next(); + try { + IOUtils.closeWhileHandlingException(next.getValue()); + connectionListener.onNodeDisconnected(next.getKey()); + } finally { + iterator.remove(); + } + } } finally { - iterator.remove(); + closeLock.writeLock().unlock(); + latch.countDown(); } - } + }); - lifecycle.moveToClosed(); + try { + try { + latch.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // ignore + } + } finally { + lifecycle.moveToClosed(); + } } private void ensureOpen() { diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index ef3d228384e4e..d90128e03a6f4 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -41,7 +41,6 @@ import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.compress.NotCompressedException; -import org.elasticsearch.common.concurrent.CompletableContext; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -209,7 +208,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements private final NamedWriteableRegistry namedWriteableRegistry; // this lock is here to make sure we close this transport and disconnect all the client nodes - // connections while no connect operations is going on... (this might help with 100% CPU when stopping the transport?) + // connections while no connect operations is going on private final ReadWriteLock closeLock = new ReentrantReadWriteLock(); protected final boolean compress; private volatile BoundTransportAddress boundAddress; @@ -344,12 +343,12 @@ public String executor() { } } - public final class NodeChannels implements Connection { + public final class NodeChannels extends CloseableConnection { private final Map typeMapping; private final List channels; private final DiscoveryNode node; private final Version version; - private final CompletableContext closeContext = new CompletableContext<>(); + private final AtomicBoolean isClosing = new AtomicBoolean(false); NodeChannels(DiscoveryNode node, List channels, ConnectionProfile connectionProfile, Version handshakeVersion) { this.node = node; @@ -410,14 +409,9 @@ protected void innerOnFailure(Exception e) { return true; } - @Override - public void addCloseListener(ActionListener listener) { - closeContext.addListener(ActionListener.toBiConsumer(listener)); - } - @Override public void close() { - if (closeContext.complete(null)) { + if (isClosing.compareAndSet(false, true)) { try { if (lifecycle.stopped()) { /* We set SO_LINGER timeout to 0 to ensure that when we shutdown the node we don't @@ -440,7 +434,8 @@ public void close() { boolean block = lifecycle.stopped() && Transports.isTransportThread(Thread.currentThread()) == false; CloseableChannel.closeChannels(channels, block); } finally { - transportListener.onConnectionClosed(this); + // Call the super method to trigger listeners + super.close(); } } } @@ -453,17 +448,12 @@ public DiscoveryNode getNode() { @Override public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { - if (closeContext.isDone()) { + if (isClosing.get()) { throw new NodeNotConnectedException(node, "connection already closed"); } TcpChannel channel = channel(options.type()); sendRequestToChannel(this.node, channel, requestId, action, request, options, getVersion(), (byte) 0); } - - @Override - public boolean isClosed() { - return closeContext.isDone(); - } } /** @@ -546,17 +536,16 @@ public NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connect // At this point we should construct the connection, notify the transport service, and attach close listeners to the // underlying channels. nodeChannels = new NodeChannels(node, channels, connectionProfile, version); - transportListener.onConnectionOpened(nodeChannels); final NodeChannels finalNodeChannels = nodeChannels; - final AtomicBoolean runOnce = new AtomicBoolean(false); + try { + transportListener.onConnectionOpened(nodeChannels); + } finally { + nodeChannels.addCloseListener(ActionListener.wrap(() -> transportListener.onConnectionClosed(finalNodeChannels))); + } + Consumer onClose = c -> { assert c.isOpen() == false : "channel is still open when onClose is called"; - // we only need to disconnect from the nodes once since all other channels - // will also try to run this we protect it from running multiple times. - if (runOnce.compareAndSet(false, true)) { - // TODO: Have lost the removal from connection manager - IOUtils.closeWhileHandlingException(finalNodeChannels); - } + finalNodeChannels.close(); }; nodeChannels.channels.forEach(ch -> ch.addCloseListener(ActionListener.wrap(() -> onClose.accept(ch)))); diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java index 99d192d9201a0..87e4d351c9672 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transport.java +++ b/server/src/main/java/org/elasticsearch/transport/Transport.java @@ -147,6 +147,8 @@ default Object getCacheKey() { return this; } + @Override + void close(); } /** diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index 95525ac8c2ba7..6885d1eea76a1 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -454,7 +454,7 @@ public boolean isClosed() { } @Override - public void close() throws IOException { + public void close() { throw new UnsupportedOperationException(); } } diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 8c18cabf7f3d4..203ea3c3e2ef8 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -453,7 +453,7 @@ public void sendRequest(long requestId, String action, TransportRequest request, } @Override - public void close() throws IOException { + public void close() { connection.close(); } diff --git a/server/src/test/java/org/elasticsearch/transport/ConnectionManagerTests.java b/server/src/test/java/org/elasticsearch/transport/ConnectionManagerTests.java new file mode 100644 index 0000000000000..64e8a42600458 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/transport/ConnectionManagerTests.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ConnectionManagerTests extends ESTestCase { + + private ConnectionManager connectionManager; + private ThreadPool threadPool; + private Transport transport; + private ConnectionProfile connectionProfile; + + @Before + public void createConnectionManager() { + Settings settings = Settings.builder() + .put("node.name", ConnectionManagerTests.class.getSimpleName()) + .build(); + threadPool = new ThreadPool(settings); + transport = mock(Transport.class); + connectionManager = new ConnectionManager(settings, transport, threadPool); + TimeValue oneSecond = new TimeValue(1000); + connectionProfile = ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG, oneSecond, oneSecond); + } + + @After + public void stopThreadPool() { + threadPool.shutdown(); + } + + public void testConnectAndDisconnect() { + AtomicInteger nodeConnectedCount = new AtomicInteger(); + AtomicInteger nodeDisconnectedCount = new AtomicInteger(); + connectionManager.addListener(new TransportConnectionListener() { + @Override + public void onNodeConnected(DiscoveryNode node) { + nodeConnectedCount.incrementAndGet(); + } + + @Override + public void onNodeDisconnected(DiscoveryNode node) { + nodeDisconnectedCount.incrementAndGet(); + } + }); + + + DiscoveryNode node = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT); + Transport.Connection connection = new TestConnect(node); + when(transport.openConnection(node, connectionProfile)).thenReturn(connection); + + assertFalse(connectionManager.nodeConnected(node)); + + AtomicReference connectionRef = new AtomicReference<>(); + CheckedBiConsumer validator = (c, p) -> connectionRef.set(c); + connectionManager.connectToNode(node, connectionProfile, validator); + + assertFalse(connection.isClosed()); + assertTrue(connectionManager.nodeConnected(node)); + assertSame(connection, connectionManager.getConnection(node)); + assertEquals(1, connectionManager.connectedNodeCount()); + assertEquals(1, nodeConnectedCount.get()); + assertEquals(0, nodeDisconnectedCount.get()); + + if (randomBoolean()) { + connectionManager.disconnectFromNode(node); + } else { + connection.close(); + } + assertTrue(connection.isClosed()); + assertEquals(0, connectionManager.connectedNodeCount()); + assertEquals(1, nodeConnectedCount.get()); + assertEquals(1, nodeDisconnectedCount.get()); + } + + public void testConnectFails() { + AtomicInteger nodeConnectedCount = new AtomicInteger(); + AtomicInteger nodeDisconnectedCount = new AtomicInteger(); + connectionManager.addListener(new TransportConnectionListener() { + @Override + public void onNodeConnected(DiscoveryNode node) { + nodeConnectedCount.incrementAndGet(); + } + + @Override + public void onNodeDisconnected(DiscoveryNode node) { + nodeDisconnectedCount.incrementAndGet(); + } + }); + + + DiscoveryNode node = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT); + Transport.Connection connection = new TestConnect(node); + when(transport.openConnection(node, connectionProfile)).thenReturn(connection); + + assertFalse(connectionManager.nodeConnected(node)); + + CheckedBiConsumer validator = (c, p) -> { + throw new ConnectTransportException(node, ""); + }; + + expectThrows(ConnectTransportException.class, () -> connectionManager.connectToNode(node, connectionProfile, validator)); + + assertTrue(connection.isClosed()); + assertFalse(connectionManager.nodeConnected(node)); + expectThrows(NodeNotConnectedException.class, () -> connectionManager.getConnection(node)); + assertEquals(0, connectionManager.connectedNodeCount()); + assertEquals(0, nodeConnectedCount.get()); + assertEquals(0, nodeDisconnectedCount.get()); + } + + private static class TestConnect extends CloseableConnection { + + private final DiscoveryNode node; + + private TestConnect(DiscoveryNode node) { + this.node = node; + } + + @Override + public DiscoveryNode getNode() { + return node; + } + + @Override + public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) + throws TransportException { + + } + } +} diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 9df80ae9e5a20..8da4064d1c89a 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -446,7 +446,7 @@ public boolean isClosed() { } @Override - public void close() throws IOException { + public void close() { // no-op } }; diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java index fcd997202aa7a..e4262ef5404b9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java @@ -232,7 +232,7 @@ public Object getCacheKey() { } @Override - public void close() throws IOException { + public void close() { connection.close(); } From 76d429c4e68dacbff979adf3a03193cdc1abc6d9 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 1 Aug 2018 18:36:16 -0600 Subject: [PATCH 26/29] Fix tests --- .../transport/ConnectionManager.java | 1 - .../transport/FailAndRetryMockTransport.java | 21 +---- .../TransportClientNodesServiceTests.java | 86 ++----------------- .../test/transport/MockTransportService.java | 6 +- .../transport/StubbableConnectionManager.java | 37 ++++++++ .../test/transport/StubbableTransport.java | 11 --- 6 files changed, 50 insertions(+), 112 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index 744d979be1fec..168c68542bbc4 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -171,7 +171,6 @@ public void close() { Map.Entry next = iterator.next(); try { IOUtils.closeWhileHandlingException(next.getValue()); - connectionListener.onNodeDisconnected(next.getKey()); } finally { iterator.remove(); } diff --git a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java index 9f45389ef6197..513f07b733cda 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java +++ b/server/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java @@ -20,7 +20,6 @@ package org.elasticsearch.client.transport; import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse; import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; @@ -31,10 +30,10 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; -import org.elasticsearch.common.concurrent.CompletableContext; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.transport.CloseableConnection; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.RequestHandlerRegistry; @@ -81,9 +80,8 @@ abstract class FailAndRetryMockTransport imp @Override public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) { - CompletableContext closeContext = new CompletableContext<>(); + return new CloseableConnection() { - return new Connection() { @Override public DiscoveryNode getNode() { return node; @@ -136,21 +134,6 @@ public void sendRequest(long requestId, String action, TransportRequest request, } } } - - @Override - public void addCloseListener(ActionListener listener) { - closeContext.addListener(ActionListener.toBiConsumer(listener)); - } - - @Override - public void close() { - closeContext.complete(null); - } - - @Override - public boolean isClosed() { - return closeContext.isDone(); - } }; } diff --git a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 203ea3c3e2ef8..afc6b47483eba 100644 --- a/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -64,7 +64,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import static org.elasticsearch.test.transport.MockTransportService.createNewService; import static org.hamcrest.CoreMatchers.equalTo; @@ -361,19 +360,13 @@ public void testSniffNodesSamplerClosesConnections() throws Exception { .build(); try (MockTransportService clientService = createNewService(clientSettings, Version.CURRENT, threadPool, null)) { - final List establishedConnections = new CopyOnWriteArrayList<>(); - final List reusedConnections = new CopyOnWriteArrayList<>(); + final List establishedConnections = new CopyOnWriteArrayList<>(); clientService.addConnectBehavior(remoteService, (transport, discoveryNode, profile) -> { - MockConnection connection = new MockConnection(transport.openConnection(discoveryNode, profile)); + Transport.Connection connection = transport.openConnection(discoveryNode, profile); establishedConnections.add(connection); return connection; }); - clientService.addGetConnectionBehavior(remoteService, (connectionManager, discoveryNode) -> { - MockConnection connection = new MockConnection(connectionManager.getConnection(discoveryNode)); - reusedConnections.add(connection); - return connection; - }); clientService.start(); @@ -383,31 +376,26 @@ public void testSniffNodesSamplerClosesConnections() throws Exception { new TransportClientNodesService(clientSettings, clientService, threadPool, (a, b) -> {})) { assertEquals(0, transportClientNodesService.connectedNodes().size()); assertEquals(0, establishedConnections.size()); - assertEquals(0, reusedConnections.size()); transportClientNodesService.addTransportAddresses(remoteService.getLocalDiscoNode().getAddress()); assertEquals(1, transportClientNodesService.connectedNodes().size()); - assertTotalConnections(establishedConnections, 2); - assertClosedConnections(establishedConnections, 1); + assertEquals(1, clientService.connectionManager().connectedNodeCount()); transportClientNodesService.doSample(); - assertTotalConnections(establishedConnections, 3); - assertOpenConnections(establishedConnections, 1); - assertClosedConnections(establishedConnections, 2); - assertTotalConnections(reusedConnections, 1); - assertOpenConnections(reusedConnections, 1); + assertEquals(1, clientService.connectionManager().connectedNodeCount()); + establishedConnections.clear(); handler.blockRequest(); Thread thread = new Thread(transportClientNodesService::doSample); thread.start(); - assertBusy(() -> assertEquals(4, establishedConnections.size())); - assertFalse("Temporary ping connection must be opened", establishedConnections.get(3).isClosed()); + assertBusy(() -> assertTrue(establishedConnections.size() >= 1)); + assertFalse("Temporary ping connection must be opened", establishedConnections.get(0).isClosed()); handler.releaseRequest(); thread.join(); - assertClosedConnections(establishedConnections, 3); + assertTrue(establishedConnections.get(0).isClosed()); } } } finally { @@ -415,64 +403,6 @@ public void testSniffNodesSamplerClosesConnections() throws Exception { } } - private void assertTotalConnections(final List connections, final int size) { - assertEquals("Expecting " + size + " total connections but got " + connections.size(), size, connections.size()); - } - - private void assertClosedConnections(final List connections, final int size) { - List closed = connections.stream().filter(MockConnection::isClosed).collect(Collectors.toList()); - assertEquals("Expecting " + size + " closed connections but got " + closed.size(), size, closed.size()); - } - - private void assertOpenConnections(final List connections, final int size) { - List open = connections.stream().filter((c) -> c.isClosed() == false).collect(Collectors.toList()); - assertEquals("Expecting " + size + " open connections but got " + open.size(), size, open.size()); - } - - class MockConnection implements Transport.Connection { - private final Transport.Connection connection; - - private MockConnection(Transport.Connection connection) { - this.connection = connection; - } - - @Override - public DiscoveryNode getNode() { - return connection.getNode(); - } - - @Override - public Version getVersion() { - return connection.getVersion(); - } - - @Override - public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws IOException, TransportException { - connection.sendRequest(requestId, action, request, options); - } - - @Override - public void close() { - connection.close(); - } - - @Override - public boolean sendPing() { - return connection.sendPing(); - } - - @Override - public void addCloseListener(ActionListener listener) { - connection.addCloseListener(listener); - } - - @Override - public boolean isClosed() { - return connection.isClosed(); - } - } - class MockHandler implements TransportRequestHandler { private final AtomicBoolean block = new AtomicBoolean(false); private final CountDownLatch release = new CountDownLatch(1); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 0326f1294169e..40a6ad6476d4a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -582,9 +582,9 @@ protected void traceRequestSent(DiscoveryNode node, long requestId, String actio } public Transport getOriginalTransport() { - StubbableTransport transport = transport(); + Transport transport = transport(); while (transport instanceof StubbableTransport) { - transport = (StubbableTransport) transport.getDelegate(); + transport = ((StubbableTransport) transport).getDelegate(); } return transport; } @@ -603,7 +603,7 @@ public Transport.Connection openConnection(DiscoveryNode node, ConnectionProfile synchronized (openConnections) { List connections = openConnections.get(node); boolean remove = connections.remove(connection); - assert remove; + assert remove : "Should have removed connection"; if (connections.isEmpty()) { openConnections.remove(node); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java index b3882801c5fc8..a74cb2752c284 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java @@ -19,12 +19,17 @@ package org.elasticsearch.test.transport; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.ConnectionManager; +import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportConnectionListener; +import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -87,6 +92,38 @@ public boolean nodeConnected(DiscoveryNode node) { return behavior.nodeConnected(delegate, node); } + @Override + public void addListener(TransportConnectionListener listener) { + delegate.addListener(listener); + } + + @Override + public void removeListener(TransportConnectionListener listener) { + delegate.removeListener(listener); + } + + @Override + public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, + CheckedBiConsumer connectionValidator) + throws ConnectTransportException { + delegate.connectToNode(node, connectionProfile, connectionValidator); + } + + @Override + public void disconnectFromNode(DiscoveryNode node) { + delegate.disconnectFromNode(node); + } + + @Override + public int connectedNodeCount() { + return delegate.connectedNodeCount(); + } + + @Override + public void close() { + delegate.close(); + } + @FunctionalInterface public interface GetConnectionBehavior { Transport.Connection getConnection(ConnectionManager connectionManager, DiscoveryNode discoveryNode); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java index e4262ef5404b9..5a0dd3b7f6d53 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java @@ -181,7 +181,6 @@ public Map profileBoundAddresses() { private class WrappedConnection implements Transport.Connection { - private final Transport.Connection connection; private WrappedConnection(Transport.Connection connection) { @@ -235,16 +234,6 @@ public Object getCacheKey() { public void close() { connection.close(); } - - @Override - public boolean equals(Object obj) { - return connection.equals(obj); - } - - @Override - public int hashCode() { - return connection.hashCode(); - } } @FunctionalInterface From 8c26686b6f9110951d809004c86daee83cdce28c Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 1 Aug 2018 18:47:21 -0600 Subject: [PATCH 27/29] java doc --- .../transport/ConnectionManager.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index 168c68542bbc4..f2b89ff59775c 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -47,6 +47,11 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +/** + * This class manages node connections. The connection is opened by the underlying transport. Once the + * connection is opened, this class manages the connection. This includes keep-alive pings and closing + * the connection when the connection manager is closed. + */ public class ConnectionManager implements Closeable { private final ConcurrentMap connectedNodes = ConcurrentCollections.newConcurrentMap(); @@ -79,6 +84,10 @@ public void removeListener(TransportConnectionListener listener) { this.connectionListener.listeners.remove(listener); } + /** + * Connects to a node with the given connection profile. If the node is already connected this method has no effect. + * Once a successful is established, it can be validated before being exposed. + */ public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, CheckedBiConsumer connectionValidator) throws ConnectTransportException { @@ -131,6 +140,14 @@ public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfil } } + /** + * Returns a connection for the given node if the node is connected. + * Connections returned from this method must not be closed. The lifecycle of this connection is + * maintained by this connection manager + * + * @throws NodeNotConnectedException if the node is not connected + * @see #connectToNode(DiscoveryNode, ConnectionProfile, CheckedBiConsumer) + */ public Transport.Connection getConnection(DiscoveryNode node) { Transport.Connection connection = connectedNodes.get(node); if (connection == null) { @@ -139,10 +156,16 @@ public Transport.Connection getConnection(DiscoveryNode node) { return connection; } + /** + * Returns {@code true} if the node is connected. + */ public boolean nodeConnected(DiscoveryNode node) { return connectedNodes.containsKey(node); } + /** + * Disconnected from the given node, if not connected, will do nothing. + */ public void disconnectFromNode(DiscoveryNode node) { Transport.Connection nodeChannels = connectedNodes.remove(node); if (nodeChannels != null) { From d18be8cf6c9fb8475426f5dbcba53ab88bf8af5b Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 13 Aug 2018 10:00:46 -0600 Subject: [PATCH 28/29] Changes from review --- .../org/elasticsearch/transport/CloseableConnection.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java b/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java index 3932fdb43485c..1fc1391edf179 100644 --- a/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java @@ -30,6 +30,13 @@ public abstract class CloseableConnection implements Transport.Connection { private final CompletableContext closeContext = new CompletableContext<>(); + /** + * The listener's {@link ActionListener#onResponse(Object)} method will be called when this connection is + * closed. This implementation does not have a scenario where the close process will be produce an + * exception, so the {@link ActionListener#onFailure(Exception)} will not be called. + * + * @param listener to be called + */ @Override public void addCloseListener(ActionListener listener) { closeContext.addListener(ActionListener.toBiConsumer(listener)); @@ -42,6 +49,8 @@ public boolean isClosed() { @Override public void close() { + // This method is safe to call multiple times as the close context will provide concurrency + // protection and only be completed once. The attached listeners will only be notified once. closeContext.complete(null); } } From f259bae4ca11ddce68a2dc59b2d5c2d4ef84eb26 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 13 Aug 2018 10:33:39 -0600 Subject: [PATCH 29/29] Changes docs --- .../org/elasticsearch/transport/CloseableConnection.java | 7 ------- .../main/java/org/elasticsearch/transport/Transport.java | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java b/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java index 1fc1391edf179..cdc68dc10f42e 100644 --- a/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/CloseableConnection.java @@ -30,13 +30,6 @@ public abstract class CloseableConnection implements Transport.Connection { private final CompletableContext closeContext = new CompletableContext<>(); - /** - * The listener's {@link ActionListener#onResponse(Object)} method will be called when this connection is - * closed. This implementation does not have a scenario where the close process will be produce an - * exception, so the {@link ActionListener#onFailure(Exception)} will not be called. - * - * @param listener to be called - */ @Override public void addCloseListener(ActionListener listener) { closeContext.addListener(ActionListener.toBiConsumer(listener)); diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java index 87e4d351c9672..9538119f43b8a 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transport.java +++ b/server/src/main/java/org/elasticsearch/transport/Transport.java @@ -128,6 +128,13 @@ default boolean sendPing() { return false; } + /** + * The listener's {@link ActionListener#onResponse(Object)} method will be called when this + * connection is closed. No implementations currently throw an exception during close, so + * {@link ActionListener#onFailure(Exception)} will not be called. + * + * @param listener to be called + */ void addCloseListener(ActionListener listener); boolean isClosed();