Skip to content

Commit e29a3ee

Browse files
committed
Introduce simple remote connection strategy (elastic#47480)
This commit introduces a simple remote connection strategy which will open remote connections to a configurable list of user supplied addresses. These addresses can be remote Elasticsearch nodes or intermediate proxies. We will perform normal clustername and version validation, but otherwise rely on the remote cluster to route requests to the appropriate remote node.
1 parent 90cbddf commit e29a3ee

File tree

5 files changed

+454
-16
lines changed

5 files changed

+454
-16
lines changed

server/src/main/java/org/elasticsearch/transport/ConnectionManager.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,10 @@
3131
import org.elasticsearch.core.internal.io.IOUtils;
3232

3333
import java.io.Closeable;
34+
import java.util.Collections;
3435
import java.util.Iterator;
3536
import java.util.Map;
37+
import java.util.Set;
3638
import java.util.concurrent.ConcurrentMap;
3739
import java.util.concurrent.CopyOnWriteArrayList;
3840
import java.util.concurrent.CountDownLatch;
@@ -216,6 +218,10 @@ public int size() {
216218
return connectedNodes.size();
217219
}
218220

221+
public Set<DiscoveryNode> getAllConnectedNodes() {
222+
return Collections.unmodifiableSet(connectedNodes.keySet());
223+
}
224+
219225
@Override
220226
public void close() {
221227
internalClose(true);

server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,17 +39,20 @@
3939

4040
public abstract class RemoteConnectionStrategy implements TransportConnectionListener, Closeable {
4141

42-
protected static final Logger logger = LogManager.getLogger(RemoteConnectionStrategy.class);
42+
private static final Logger logger = LogManager.getLogger(RemoteConnectionStrategy.class);
4343

4444
private static final int MAX_LISTENERS = 100;
4545
private final AtomicBoolean closed = new AtomicBoolean(false);
4646
private final Object mutex = new Object();
47-
private final ThreadPool threadPool;
48-
protected final RemoteConnectionManager connectionManager;
4947
private List<ActionListener<Void>> listeners = new ArrayList<>();
5048

51-
RemoteConnectionStrategy(ThreadPool threadPool, RemoteConnectionManager connectionManager) {
52-
this.threadPool = threadPool;
49+
protected final TransportService transportService;
50+
protected final RemoteConnectionManager connectionManager;
51+
protected final String clusterAlias;
52+
53+
RemoteConnectionStrategy(String clusterAlias, TransportService transportService, RemoteConnectionManager connectionManager) {
54+
this.clusterAlias = clusterAlias;
55+
this.transportService = transportService;
5356
this.connectionManager = connectionManager;
5457
connectionManager.getConnectionManager().addListener(this);
5558
}
@@ -61,7 +64,7 @@ public abstract class RemoteConnectionStrategy implements TransportConnectionLis
6164
void connect(ActionListener<Void> connectListener) {
6265
boolean runConnect = false;
6366
final ActionListener<Void> listener =
64-
ContextPreservingActionListener.wrapPreservingContext(connectListener, threadPool.getThreadContext());
67+
ContextPreservingActionListener.wrapPreservingContext(connectListener, transportService.getThreadPool().getThreadContext());
6568
boolean closed;
6669
synchronized (mutex) {
6770
closed = this.closed.get();
@@ -83,7 +86,7 @@ void connect(ActionListener<Void> connectListener) {
8386
return;
8487
}
8588
if (runConnect) {
86-
ExecutorService executor = threadPool.executor(ThreadPool.Names.MANAGEMENT);
89+
ExecutorService executor = transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT);
8790
executor.submit(new AbstractRunnable() {
8891
@Override
8992
public void onFailure(Exception e) {
Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.transport;
21+
22+
import org.apache.logging.log4j.LogManager;
23+
import org.apache.logging.log4j.Logger;
24+
import org.apache.logging.log4j.message.ParameterizedMessage;
25+
import org.elasticsearch.Version;
26+
import org.elasticsearch.action.ActionListener;
27+
import org.elasticsearch.cluster.ClusterName;
28+
import org.elasticsearch.cluster.node.DiscoveryNode;
29+
import org.elasticsearch.common.transport.TransportAddress;
30+
import org.elasticsearch.common.util.concurrent.CountDown;
31+
32+
import java.util.Iterator;
33+
import java.util.List;
34+
import java.util.concurrent.atomic.AtomicInteger;
35+
import java.util.concurrent.atomic.AtomicLong;
36+
import java.util.concurrent.atomic.AtomicReference;
37+
import java.util.function.Supplier;
38+
import java.util.stream.Collectors;
39+
40+
public class SimpleConnectionStrategy extends RemoteConnectionStrategy {
41+
42+
private static final int MAX_CONNECT_ATTEMPTS_PER_RUN = 3;
43+
private static final Logger logger = LogManager.getLogger(SimpleConnectionStrategy.class);
44+
45+
private final int maxNumRemoteConnections;
46+
private final AtomicLong counter = new AtomicLong(0);
47+
private final List<Supplier<TransportAddress>> addresses;
48+
private final AtomicReference<ClusterName> remoteClusterName = new AtomicReference<>();
49+
private final ConnectionProfile profile;
50+
private final ConnectionManager.ConnectionValidator clusterNameValidator;
51+
52+
SimpleConnectionStrategy(String clusterAlias, TransportService transportService, RemoteConnectionManager connectionManager,
53+
int maxNumRemoteConnections, List<Supplier<TransportAddress>> addresses) {
54+
super(clusterAlias, transportService, connectionManager);
55+
this.maxNumRemoteConnections = maxNumRemoteConnections;
56+
assert addresses.isEmpty() == false : "Cannot use simple connection strategy with no configured addresses";
57+
this.addresses = addresses;
58+
// TODO: Move into the ConnectionManager
59+
this.profile = new ConnectionProfile.Builder()
60+
.addConnections(1, TransportRequestOptions.Type.REG, TransportRequestOptions.Type.PING)
61+
.addConnections(0, TransportRequestOptions.Type.BULK, TransportRequestOptions.Type.STATE, TransportRequestOptions.Type.RECOVERY)
62+
.build();
63+
this.clusterNameValidator = (newConnection, actualProfile, listener) ->
64+
transportService.handshake(newConnection, actualProfile.getHandshakeTimeout().millis(), cn -> true,
65+
ActionListener.map(listener, resp -> {
66+
ClusterName remote = resp.getClusterName();
67+
if (remoteClusterName.compareAndSet(null, remote)) {
68+
return null;
69+
} else {
70+
if (remoteClusterName.get().equals(remote) == false) {
71+
DiscoveryNode node = newConnection.getNode();
72+
throw new ConnectTransportException(node, "handshake failed. unexpected remote cluster name " + remote);
73+
}
74+
return null;
75+
}
76+
}));
77+
}
78+
79+
@Override
80+
protected boolean shouldOpenMoreConnections() {
81+
return connectionManager.size() < maxNumRemoteConnections;
82+
}
83+
84+
@Override
85+
protected void connectImpl(ActionListener<Void> listener) {
86+
performSimpleConnectionProcess(addresses.iterator(), listener);
87+
}
88+
89+
private void performSimpleConnectionProcess(Iterator<Supplier<TransportAddress>> addressIter, ActionListener<Void> listener) {
90+
openConnections(listener, 1);
91+
}
92+
93+
private void openConnections(ActionListener<Void> finished, int attemptNumber) {
94+
if (attemptNumber <= MAX_CONNECT_ATTEMPTS_PER_RUN) {
95+
List<TransportAddress> resolved = addresses.stream().map(Supplier::get).collect(Collectors.toList());
96+
97+
int remaining = maxNumRemoteConnections - connectionManager.size();
98+
ActionListener<Void> compositeListener = new ActionListener<>() {
99+
100+
private final AtomicInteger successfulConnections = new AtomicInteger(0);
101+
private final CountDown countDown = new CountDown(remaining);
102+
103+
@Override
104+
public void onResponse(Void v) {
105+
successfulConnections.incrementAndGet();
106+
if (countDown.countDown()) {
107+
if (shouldOpenMoreConnections()) {
108+
openConnections(finished, attemptNumber + 1);
109+
} else {
110+
finished.onResponse(v);
111+
}
112+
}
113+
}
114+
115+
@Override
116+
public void onFailure(Exception e) {
117+
if (countDown.countDown()) {
118+
openConnections(finished, attemptNumber + 1);
119+
}
120+
}
121+
};
122+
123+
124+
for (int i = 0; i < remaining; ++i) {
125+
TransportAddress address = nextAddress(resolved);
126+
String id = clusterAlias + "#" + address;
127+
DiscoveryNode node = new DiscoveryNode(id, address, Version.CURRENT.minimumCompatibilityVersion());
128+
129+
connectionManager.connectToNode(node, profile, clusterNameValidator, new ActionListener<>() {
130+
@Override
131+
public void onResponse(Void v) {
132+
compositeListener.onResponse(v);
133+
}
134+
135+
@Override
136+
public void onFailure(Exception e) {
137+
logger.debug(new ParameterizedMessage("failed to open remote connection [remote cluster: {}, address: {}]",
138+
clusterAlias, address), e);
139+
compositeListener.onFailure(e);
140+
}
141+
});
142+
}
143+
} else {
144+
int openConnections = connectionManager.size();
145+
if (openConnections == 0) {
146+
finished.onFailure(new IllegalStateException("Unable to open any simple connections to remote cluster [" + clusterAlias
147+
+ "]"));
148+
} else {
149+
logger.debug("unable to open maximum number of connections [remote cluster: {}, opened: {}, maximum: {}]", clusterAlias,
150+
openConnections, maxNumRemoteConnections);
151+
finished.onResponse(null);
152+
}
153+
}
154+
}
155+
156+
private TransportAddress nextAddress(List<TransportAddress> resolvedAddresses) {
157+
long curr;
158+
while ((curr = counter.getAndIncrement()) == Long.MIN_VALUE) ;
159+
return resolvedAddresses.get(Math.floorMod(curr, resolvedAddresses.size()));
160+
}
161+
}

server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
package org.elasticsearch.transport;
2121

22+
import org.apache.logging.log4j.LogManager;
23+
import org.apache.logging.log4j.Logger;
2224
import org.apache.logging.log4j.message.ParameterizedMessage;
2325
import org.apache.lucene.util.SetOnce;
2426
import org.elasticsearch.action.ActionListener;
@@ -45,9 +47,9 @@
4547

4648
public class SniffConnectionStrategy extends RemoteConnectionStrategy {
4749

48-
private final String clusterAlias;
50+
private static final Logger logger = LogManager.getLogger(SniffConnectionStrategy.class);
51+
4952
private final List<Tuple<String, Supplier<DiscoveryNode>>> seedNodes;
50-
private final TransportService transportService;
5153
private final int maxNumRemoteConnections;
5254
private final Predicate<DiscoveryNode> nodePredicate;
5355
private final SetOnce<ClusterName> remoteClusterName = new SetOnce<>();
@@ -56,9 +58,7 @@ public class SniffConnectionStrategy extends RemoteConnectionStrategy {
5658
SniffConnectionStrategy(String clusterAlias, TransportService transportService, RemoteConnectionManager connectionManager,
5759
String proxyAddress, int maxNumRemoteConnections, Predicate<DiscoveryNode> nodePredicate,
5860
List<Tuple<String, Supplier<DiscoveryNode>>> seedNodes) {
59-
super(transportService.getThreadPool(), connectionManager);
60-
this.clusterAlias = clusterAlias;
61-
this.transportService = transportService;
61+
super(clusterAlias, transportService, connectionManager);
6262
this.proxyAddress = proxyAddress;
6363
this.maxNumRemoteConnections = maxNumRemoteConnections;
6464
this.nodePredicate = nodePredicate;
@@ -109,15 +109,15 @@ private void collectRemoteNodes(Iterator<Supplier<DiscoveryNode>> seedNodes, Act
109109
onFailure.accept(e);
110110
}
111111

112-
final StepListener<TransportService.HandshakeResponse> handShakeStep = new StepListener<>();
112+
final StepListener<TransportService.HandshakeResponse> handshakeStep = new StepListener<>();
113113
openConnectionStep.whenComplete(connection -> {
114114
ConnectionProfile connectionProfile = connectionManager.getConnectionManager().getConnectionProfile();
115115
transportService.handshake(connection, connectionProfile.getHandshakeTimeout().millis(),
116-
getRemoteClusterNamePredicate(), handShakeStep);
116+
getRemoteClusterNamePredicate(), handshakeStep);
117117
}, onFailure);
118118

119119
final StepListener<Void> fullConnectionStep = new StepListener<>();
120-
handShakeStep.whenComplete(handshakeResponse -> {
120+
handshakeStep.whenComplete(handshakeResponse -> {
121121
final DiscoveryNode handshakeNode = maybeAddProxyAddress(proxyAddress, handshakeResponse.getDiscoveryNode());
122122

123123
if (nodePredicate.test(handshakeNode) && shouldOpenMoreConnections()) {
@@ -135,7 +135,7 @@ private void collectRemoteNodes(Iterator<Supplier<DiscoveryNode>> seedNodes, Act
135135

136136
fullConnectionStep.whenComplete(aVoid -> {
137137
if (remoteClusterName.get() == null) {
138-
TransportService.HandshakeResponse handshakeResponse = handShakeStep.result();
138+
TransportService.HandshakeResponse handshakeResponse = handshakeStep.result();
139139
assert handshakeResponse.getClusterName().value() != null;
140140
remoteClusterName.set(handshakeResponse.getClusterName());
141141
}

0 commit comments

Comments
 (0)