Skip to content

Commit 6bae215

Browse files
Tim-BrooksTim-Brooks
committed
Wildcard cluster names for cross cluster search (#23985)
This is related to #23893. This commit allows users to use wilcards for cluster names when executing a cross cluster search. So instead of defining every cluster such as: GET one:*,two:*,three:*/_search A user could just search: GET *:*/_search As ":" characters are currently allowed in index names, if the text up to the first ":" does not match a defined cluster name, the entire string is treated as an index name.
1 parent 2da4b68 commit 6bae215

File tree

5 files changed

+218
-10
lines changed

5 files changed

+218
-10
lines changed

‎core/src/main/java/org/elasticsearch/action/search/RemoteClusterService.java‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
2727
import org.elasticsearch.action.support.GroupedActionListener;
2828
import org.elasticsearch.action.support.PlainActionFuture;
29+
import org.elasticsearch.cluster.metadata.ClusterNameExpressionResolver;
2930
import org.elasticsearch.cluster.node.DiscoveryNode;
3031
import org.elasticsearch.cluster.routing.PlainShardIterator;
3132
import org.elasticsearch.cluster.routing.ShardIterator;
@@ -59,6 +60,7 @@
5960
import java.util.HashMap;
6061
import java.util.List;
6162
import java.util.Map;
63+
import java.util.Set;
6264
import java.util.concurrent.ConcurrentHashMap;
6365
import java.util.concurrent.TimeUnit;
6466
import java.util.concurrent.TimeoutException;
@@ -115,11 +117,13 @@ public final class RemoteClusterService extends AbstractComponent implements Clo
115117

116118
private final TransportService transportService;
117119
private final int numRemoteConnections;
120+
private final ClusterNameExpressionResolver clusterNameResolver;
118121
private volatile Map<String, RemoteClusterConnection> remoteClusters = Collections.emptyMap();
119122

120123
RemoteClusterService(Settings settings, TransportService transportService) {
121124
super(settings);
122125
this.transportService = transportService;
126+
this.clusterNameResolver = new ClusterNameExpressionResolver(settings);
123127
numRemoteConnections = REMOTE_CONNECTIONS_PER_CLUSTER.get(settings);
124128
}
125129

@@ -216,25 +220,30 @@ boolean isRemoteNodeConnected(final String remoteCluster, final DiscoveryNode no
216220
*/
217221
Map<String, List<String>> groupClusterIndices(String[] requestIndices, Predicate<String> indexExists) {
218222
Map<String, List<String>> perClusterIndices = new HashMap<>();
223+
Set<String> remoteClusterNames = this.remoteClusters.keySet();
219224
for (String index : requestIndices) {
220225
int i = index.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR);
221-
String indexName = index;
222-
String clusterName = LOCAL_CLUSTER_GROUP_KEY;
223226
if (i >= 0) {
224227
String remoteClusterName = index.substring(0, i);
225-
if (isRemoteClusterRegistered(remoteClusterName)) {
228+
List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusterNames, remoteClusterName);
229+
if (clusters.isEmpty() == false) {
226230
if (indexExists.test(index)) {
227231
// we use : as a separator for remote clusters. might conflict if there is an index that is actually named
228232
// remote_cluster_alias:index_name - for this case we fail the request. the user can easily change the cluster alias
229233
// if that happens
230234
throw new IllegalArgumentException("Can not filter indices; index " + index +
231235
" exists but there is also a remote cluster named: " + remoteClusterName);
236+
}
237+
String indexName = index.substring(i + 1);
238+
for (String clusterName : clusters) {
239+
perClusterIndices.computeIfAbsent(clusterName, k -> new ArrayList<>()).add(indexName);
232240
}
233-
indexName = index.substring(i + 1);
234-
clusterName = remoteClusterName;
241+
} else {
242+
perClusterIndices.computeIfAbsent(LOCAL_CLUSTER_GROUP_KEY, k -> new ArrayList<>()).add(index);
235243
}
244+
} else {
245+
perClusterIndices.computeIfAbsent(LOCAL_CLUSTER_GROUP_KEY, k -> new ArrayList<>()).add(index);
236246
}
237-
perClusterIndices.computeIfAbsent(clusterName, k -> new ArrayList<String>()).add(indexName);
238247
}
239248
return perClusterIndices;
240249
}

‎core/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.cluster.metadata;
21+
22+
import org.elasticsearch.common.component.AbstractComponent;
23+
import org.elasticsearch.common.regex.Regex;
24+
import org.elasticsearch.common.settings.Settings;
25+
26+
import java.util.ArrayList;
27+
import java.util.Collections;
28+
import java.util.List;
29+
import java.util.Set;
30+
import java.util.stream.Collectors;
31+
32+
/**
33+
* Resolves cluster names from an expression. The expression must be the exact match of a cluster
34+
* name or must be a wildcard expression.
35+
*/
36+
public final class ClusterNameExpressionResolver extends AbstractComponent {
37+
38+
private final WildcardExpressionResolver wildcardResolver = new WildcardExpressionResolver();
39+
40+
public ClusterNameExpressionResolver(Settings settings) {
41+
super(settings);
42+
}
43+
44+
/**
45+
* Resolves the provided cluster expression to matching cluster names. This method only
46+
* supports exact or wildcard matches.
47+
*
48+
* @param remoteClusters the aliases for remote clusters
49+
* @param clusterExpression the expressions that can be resolved to cluster names.
50+
* @return the resolved cluster aliases.
51+
*/
52+
public List<String> resolveClusterNames(Set<String> remoteClusters, String clusterExpression) {
53+
if (remoteClusters.contains(clusterExpression)) {
54+
return Collections.singletonList(clusterExpression);
55+
} else if (Regex.isSimpleMatchPattern(clusterExpression)) {
56+
return wildcardResolver.resolve(remoteClusters, clusterExpression);
57+
} else {
58+
return Collections.emptyList();
59+
}
60+
}
61+
62+
private static class WildcardExpressionResolver {
63+
64+
private List<String> resolve(Set<String> remoteClusters, String clusterExpression) {
65+
if (isTrivialWildcard(clusterExpression)) {
66+
return resolveTrivialWildcard(remoteClusters);
67+
}
68+
69+
Set<String> matches = matches(remoteClusters, clusterExpression);
70+
if (matches.isEmpty()) {
71+
return Collections.emptyList();
72+
} else {
73+
return new ArrayList<>(matches);
74+
}
75+
}
76+
77+
private boolean isTrivialWildcard(String clusterExpression) {
78+
return Regex.isMatchAllPattern(clusterExpression);
79+
}
80+
81+
private List<String> resolveTrivialWildcard(Set<String> remoteClusters) {
82+
return new ArrayList<>(remoteClusters);
83+
}
84+
85+
private static Set<String> matches(Set<String> remoteClusters, String expression) {
86+
if (expression.indexOf("*") == expression.length() - 1) {
87+
return otherWildcard(remoteClusters, expression);
88+
} else {
89+
return otherWildcard(remoteClusters, expression);
90+
}
91+
}
92+
93+
private static Set<String> otherWildcard(Set<String> remoteClusters, String expression) {
94+
final String pattern = expression;
95+
return remoteClusters.stream()
96+
.filter(n -> Regex.simpleMatch(pattern, n))
97+
.collect(Collectors.toSet());
98+
}
99+
}
100+
}

‎core/src/test/java/org/elasticsearch/action/search/RemoteClusterServiceTests.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,14 +148,14 @@ public void testGroupClusterIndices() throws IOException {
148148
assertTrue(service.isRemoteClusterRegistered("cluster_2"));
149149
assertFalse(service.isRemoteClusterRegistered("foo"));
150150
Map<String, List<String>> perClusterIndices = service.groupClusterIndices(new String[]{"foo:bar", "cluster_1:bar",
151-
"cluster_2:foo:bar", "cluster_1:test", "cluster_2:foo*", "foo"}, i -> false);
151+
"cluster_2:foo:bar", "cluster_1:test", "cluster_2:foo*", "foo", "cluster*:baz", "*:boo", "no*match:boo"}, i -> false);
152152
String[] localIndices = perClusterIndices.computeIfAbsent(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY,
153153
k -> Collections.emptyList()).toArray(new String[0]);
154154
assertNotNull(perClusterIndices.remove(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY));
155-
assertArrayEquals(new String[]{"foo:bar", "foo"}, localIndices);
155+
assertArrayEquals(new String[]{"foo:bar", "foo", "no*match:boo"}, localIndices);
156156
assertEquals(2, perClusterIndices.size());
157-
assertEquals(Arrays.asList("bar", "test"), perClusterIndices.get("cluster_1"));
158-
assertEquals(Arrays.asList("foo:bar", "foo*"), perClusterIndices.get("cluster_2"));
157+
assertEquals(Arrays.asList("bar", "test", "baz", "boo"), perClusterIndices.get("cluster_1"));
158+
assertEquals(Arrays.asList("foo:bar", "foo*", "baz", "boo"), perClusterIndices.get("cluster_2"));
159159

160160
IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
161161
service.groupClusterIndices(new String[]{"foo:bar", "cluster_1:bar",

‎core/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.cluster.metadata;
21+
22+
import org.elasticsearch.common.settings.Settings;
23+
import org.elasticsearch.test.ESTestCase;
24+
25+
import java.util.Arrays;
26+
import java.util.HashSet;
27+
import java.util.List;
28+
import java.util.Set;
29+
30+
public class ClusterNameExpressionResolverTests extends ESTestCase {
31+
32+
private ClusterNameExpressionResolver clusterNameResolver = new ClusterNameExpressionResolver(Settings.EMPTY);
33+
private static final Set<String> remoteClusters = new HashSet<>();
34+
35+
static {
36+
remoteClusters.add("cluster1");
37+
remoteClusters.add("cluster2");
38+
remoteClusters.add("totallyDifferent");
39+
}
40+
41+
public void testExactMatch() {
42+
List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "totallyDifferent");
43+
assertEquals(new HashSet<>(Arrays.asList("totallyDifferent")), new HashSet<>(clusters));
44+
}
45+
46+
public void testNoWildCardNoMatch() {
47+
List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "totallyDifferent2");
48+
assertTrue(clusters.isEmpty());
49+
}
50+
51+
public void testWildCardNoMatch() {
52+
List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "totally*2");
53+
assertTrue(clusters.isEmpty());
54+
}
55+
56+
public void testSimpleWildCard() {
57+
List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "*");
58+
assertEquals(new HashSet<>(Arrays.asList("cluster1", "cluster2", "totallyDifferent")), new HashSet<>(clusters));
59+
}
60+
61+
public void testSuffixWildCard() {
62+
List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "cluster*");
63+
assertEquals(new HashSet<>(Arrays.asList("cluster1", "cluster2")), new HashSet<>(clusters));
64+
}
65+
66+
public void testPrefixWildCard() {
67+
List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "*Different");
68+
assertEquals(new HashSet<>(Arrays.asList("totallyDifferent")), new HashSet<>(clusters));
69+
}
70+
71+
public void testMiddleWildCard() {
72+
List<String> clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "clu*1");
73+
assertEquals(new HashSet<>(Arrays.asList("cluster1")), new HashSet<>(clusters));
74+
}
75+
}

‎qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,30 @@
118118
- match: { hits.total: 6 }
119119
- match: { hits.hits.0._index: "test_remote_cluster:test_index" }
120120

121+
---
122+
"Test wildcard search":
123+
- do:
124+
cluster.get_settings:
125+
include_defaults: true
126+
127+
- set: { defaults.search.remote.my_remote_cluster.seeds.0: remote_ip }
128+
129+
- do:
130+
cluster.put_settings:
131+
flat_settings: true
132+
body:
133+
transient:
134+
search.remote.test_remote_cluster.seeds: $remote_ip
135+
136+
- match: {transient: {search.remote.test_remote_cluster.seeds: $remote_ip}}
137+
138+
- do:
139+
search:
140+
index: "*:test_index"
141+
142+
- match: { _shards.total: 6 }
143+
- match: { hits.total: 12 }
144+
121145
---
122146
"Search an filtered alias on the remote cluster":
123147

0 commit comments

Comments
 (0)