Skip to content

Commit 11e742a

Browse files
martijnvgkcm
martijnvg
authored and
kcm
committed
[CCR] Move headers from auto follow pattern to auto follow metadata (#33846)
This ensures that we will not serialize the headers as part of the auto follow pattern in the to be added get auto follow api.
1 parent 0a5ef93 commit 11e742a

File tree

9 files changed

+119
-83
lines changed

9 files changed

+119
-83
lines changed

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,8 @@ void autoFollowIndices() {
254254
final String clusterAlias = entry.getKey();
255255
final AutoFollowPattern autoFollowPattern = entry.getValue();
256256

257-
getLeaderClusterState(autoFollowPattern.getHeaders(), clusterAlias, (leaderClusterState, e) -> {
257+
Map<String, String> headers = autoFollowMetadata.getHeaders().get(clusterAlias);
258+
getLeaderClusterState(headers, clusterAlias, (leaderClusterState, e) -> {
258259
if (leaderClusterState != null) {
259260
assert e == null;
260261
final List<String> followedIndices = autoFollowMetadata.getFollowedLeaderIndexUUIDs().get(clusterAlias);
@@ -264,7 +265,7 @@ void autoFollowIndices() {
264265
finalise(slot, new AutoFollowResult(clusterAlias));
265266
} else {
266267
Consumer<AutoFollowResult> resultHandler = result -> finalise(slot, result);
267-
checkAutoFollowPattern(clusterAlias, autoFollowPattern, leaderIndicesToFollow, resultHandler);
268+
checkAutoFollowPattern(clusterAlias, autoFollowPattern, leaderIndicesToFollow, headers, resultHandler);
268269
}
269270
} else {
270271
finalise(slot, new AutoFollowResult(clusterAlias, e));
@@ -274,15 +275,18 @@ void autoFollowIndices() {
274275
}
275276
}
276277

277-
private void checkAutoFollowPattern(String clusterAlias, AutoFollowPattern autoFollowPattern,
278-
List<Index> leaderIndicesToFollow, Consumer<AutoFollowResult> resultHandler) {
278+
private void checkAutoFollowPattern(String clusterAlias,
279+
AutoFollowPattern autoFollowPattern,
280+
List<Index> leaderIndicesToFollow,
281+
Map<String, String> headers,
282+
Consumer<AutoFollowResult> resultHandler) {
279283

280284
final CountDown leaderIndicesCountDown = new CountDown(leaderIndicesToFollow.size());
281285
final AtomicArray<Tuple<Index, Exception>> results = new AtomicArray<>(leaderIndicesToFollow.size());
282286
for (int i = 0; i < leaderIndicesToFollow.size(); i++) {
283287
final Index indexToFollow = leaderIndicesToFollow.get(i);
284288
final int slot = i;
285-
followLeaderIndex(clusterAlias, indexToFollow, autoFollowPattern, error -> {
289+
followLeaderIndex(clusterAlias, indexToFollow, autoFollowPattern, headers, error -> {
286290
results.set(slot, new Tuple<>(indexToFollow, error));
287291
if (leaderIndicesCountDown.countDown()) {
288292
resultHandler.accept(new AutoFollowResult(clusterAlias, results.asList()));
@@ -291,8 +295,11 @@ private void checkAutoFollowPattern(String clusterAlias, AutoFollowPattern autoF
291295
}
292296
}
293297

294-
private void followLeaderIndex(String clusterAlias, Index indexToFollow,
295-
AutoFollowPattern pattern, Consumer<Exception> onResult) {
298+
private void followLeaderIndex(String clusterAlias,
299+
Index indexToFollow,
300+
AutoFollowPattern pattern,
301+
Map<String,String> headers,
302+
Consumer<Exception> onResult) {
296303
final String leaderIndexName = indexToFollow.getName();
297304
final String followIndexName = getFollowerIndexName(pattern, leaderIndexName);
298305

@@ -319,7 +326,7 @@ private void followLeaderIndex(String clusterAlias, Index indexToFollow,
319326
// The coordinator always runs on the elected master node, so we can update cluster state here:
320327
updateAutoFollowMetadata(function, onResult);
321328
};
322-
createAndFollow(pattern.getHeaders(), request, successHandler, onResult);
329+
createAndFollow(headers, request, successHandler, onResult);
323330
}
324331

325332
private void finalise(int slot, AutoFollowResult result) {
@@ -357,7 +364,8 @@ static String getFollowerIndexName(AutoFollowPattern autoFollowPattern, String l
357364
}
358365
}
359366

360-
static Function<ClusterState, ClusterState> recordLeaderIndexAsFollowFunction(String clusterAlias, Index indexToFollow) {
367+
static Function<ClusterState, ClusterState> recordLeaderIndexAsFollowFunction(String clusterAlias,
368+
Index indexToFollow) {
361369
return currentState -> {
362370
AutoFollowMetadata currentAutoFollowMetadata = currentState.metaData().custom(AutoFollowMetadata.TYPE);
363371

@@ -366,8 +374,8 @@ static Function<ClusterState, ClusterState> recordLeaderIndexAsFollowFunction(St
366374
newFollowedIndexUUIDS.get(clusterAlias).add(indexToFollow.getUUID());
367375

368376
ClusterState.Builder newState = ClusterState.builder(currentState);
369-
AutoFollowMetadata newAutoFollowMetadata =
370-
new AutoFollowMetadata(currentAutoFollowMetadata.getPatterns(), newFollowedIndexUUIDS);
377+
AutoFollowMetadata newAutoFollowMetadata = new AutoFollowMetadata(currentAutoFollowMetadata.getPatterns(),
378+
newFollowedIndexUUIDS, currentAutoFollowMetadata.getHeaders());
371379
newState.metaData(MetaData.builder(currentState.getMetaData())
372380
.putCustom(AutoFollowMetadata.TYPE, newAutoFollowMetadata)
373381
.build());

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,12 @@ static ClusterState innerDelete(DeleteAutoFollowPatternAction.Request request, C
8585
final Map<String, AutoFollowPattern> patternsCopy = new HashMap<>(patterns);
8686
final Map<String, List<String>> followedLeaderIndexUUIDSCopy =
8787
new HashMap<>(currentAutoFollowMetadata.getFollowedLeaderIndexUUIDs());
88+
final Map<String, Map<String, String>> headers = new HashMap<>(currentAutoFollowMetadata.getHeaders());
8889
patternsCopy.remove(request.getLeaderClusterAlias());
8990
followedLeaderIndexUUIDSCopy.remove(request.getLeaderClusterAlias());
91+
headers.remove(request.getLeaderClusterAlias());
9092

91-
AutoFollowMetadata newAutoFollowMetadata = new AutoFollowMetadata(patternsCopy, followedLeaderIndexUUIDSCopy);
93+
AutoFollowMetadata newAutoFollowMetadata = new AutoFollowMetadata(patternsCopy, followedLeaderIndexUUIDSCopy, headers);
9294
ClusterState.Builder newState = ClusterState.builder(currentState);
9395
newState.metaData(MetaData.builder(currentState.getMetaData())
9496
.putCustom(AutoFollowMetadata.TYPE, newAutoFollowMetadata)

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -123,12 +123,15 @@ static ClusterState innerPut(PutAutoFollowPatternAction.Request request,
123123
AutoFollowMetadata currentAutoFollowMetadata = localState.metaData().custom(AutoFollowMetadata.TYPE);
124124
Map<String, List<String>> followedLeaderIndices;
125125
Map<String, AutoFollowPattern> patterns;
126+
Map<String, Map<String, String>> headers;
126127
if (currentAutoFollowMetadata != null) {
127128
patterns = new HashMap<>(currentAutoFollowMetadata.getPatterns());
128129
followedLeaderIndices = new HashMap<>(currentAutoFollowMetadata.getFollowedLeaderIndexUUIDs());
130+
headers = new HashMap<>(currentAutoFollowMetadata.getHeaders());
129131
} else {
130132
patterns = new HashMap<>();
131133
followedLeaderIndices = new HashMap<>();
134+
headers = new HashMap<>();
132135
}
133136

134137
AutoFollowPattern previousPattern = patterns.get(request.getLeaderClusterAlias());
@@ -147,6 +150,10 @@ static ClusterState innerPut(PutAutoFollowPatternAction.Request request,
147150
followedIndexUUIDs);
148151
}
149152

153+
if (filteredHeaders != null) {
154+
headers.put(request.getLeaderClusterAlias(), filteredHeaders);
155+
}
156+
150157
AutoFollowPattern autoFollowPattern = new AutoFollowPattern(
151158
request.getLeaderIndexPatterns(),
152159
request.getFollowIndexNamePattern(),
@@ -156,12 +163,11 @@ static ClusterState innerPut(PutAutoFollowPatternAction.Request request,
156163
request.getMaxConcurrentWriteBatches(),
157164
request.getMaxWriteBufferSize(),
158165
request.getMaxRetryDelay(),
159-
request.getPollTimeout(),
160-
filteredHeaders);
166+
request.getPollTimeout());
161167
patterns.put(request.getLeaderClusterAlias(), autoFollowPattern);
162168
ClusterState.Builder newState = ClusterState.builder(localState);
163169
newState.metaData(MetaData.builder(localState.getMetaData())
164-
.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, followedLeaderIndices))
170+
.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, followedLeaderIndices, headers))
165171
.build());
166172
return newState.build();
167173
}

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowMetadataTests.java‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313

1414
import java.io.IOException;
1515
import java.util.Arrays;
16-
import java.util.Collections;
1716
import java.util.HashMap;
1817
import java.util.List;
1918
import java.util.Map;
@@ -36,6 +35,7 @@ protected AutoFollowMetadata createTestInstance() {
3635
int numEntries = randomIntBetween(0, 32);
3736
Map<String, AutoFollowMetadata.AutoFollowPattern> configs = new HashMap<>(numEntries);
3837
Map<String, List<String>> followedLeaderIndices = new HashMap<>(numEntries);
38+
Map<String, Map<String, String>> headers = new HashMap<>(numEntries);
3939
for (int i = 0; i < numEntries; i++) {
4040
List<String> leaderPatterns = Arrays.asList(generateRandomStringArray(4, 4, false));
4141
AutoFollowMetadata.AutoFollowPattern autoFollowPattern = new AutoFollowMetadata.AutoFollowPattern(
@@ -47,12 +47,19 @@ protected AutoFollowMetadata createTestInstance() {
4747
randomIntBetween(0, Integer.MAX_VALUE),
4848
randomIntBetween(0, Integer.MAX_VALUE),
4949
TimeValue.timeValueMillis(500),
50-
TimeValue.timeValueMillis(500),
51-
randomBoolean() ? null : Collections.singletonMap("key", "value"));
50+
TimeValue.timeValueMillis(500));
5251
configs.put(Integer.toString(i), autoFollowPattern);
5352
followedLeaderIndices.put(Integer.toString(i), Arrays.asList(generateRandomStringArray(4, 4, false)));
53+
if (randomBoolean()) {
54+
int numHeaderEntries = randomIntBetween(1, 16);
55+
Map<String, String> header = new HashMap<>();
56+
for (int j = 0; j < numHeaderEntries; j++) {
57+
header.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
58+
}
59+
headers.put(Integer.toString(i), header);
60+
}
5461
}
55-
return new AutoFollowMetadata(configs, followedLeaderIndices);
62+
return new AutoFollowMetadata(configs, followedLeaderIndices, headers);
5663
}
5764

5865
@Override

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -140,11 +140,11 @@ public void testAutoFollowCoordinatorLogsSkippingAutoFollowCoordinationWithNonCo
140140
@Override
141141
public ClusterState execute(ClusterState currentState) throws Exception {
142142
AutoFollowPattern autoFollowPattern =
143-
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null, null);
143+
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null);
144144
AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(
145145
Collections.singletonMap("test_alias", autoFollowPattern),
146-
Collections.emptyMap()
147-
);
146+
Collections.emptyMap(),
147+
Collections.emptyMap());
148148

149149
ClusterState.Builder newState = ClusterState.builder(currentState);
150150
newState.metaData(MetaData.builder(currentState.getMetaData())

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,14 @@ public void testAutoFollower() {
5656
.build();
5757

5858
AutoFollowPattern autoFollowPattern =
59-
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null, null);
59+
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null);
6060
Map<String, AutoFollowPattern> patterns = new HashMap<>();
6161
patterns.put("remote", autoFollowPattern);
6262
Map<String, List<String>> followedLeaderIndexUUIDS = new HashMap<>();
6363
followedLeaderIndexUUIDS.put("remote", new ArrayList<>());
64-
AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(patterns, followedLeaderIndexUUIDS);
64+
Map<String, Map<String, String>> autoFollowHeaders = new HashMap<>();
65+
autoFollowHeaders.put("remote", Collections.singletonMap("key", "val"));
66+
AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(patterns, followedLeaderIndexUUIDS, autoFollowHeaders);
6567

6668
ClusterState currentState = ClusterState.builder(new ClusterName("name"))
6769
.metaData(MetaData.builder().putCustom(AutoFollowMetadata.TYPE, autoFollowMetadata))
@@ -83,6 +85,7 @@ public void testAutoFollower() {
8385
void getLeaderClusterState(Map<String, String> headers,
8486
String leaderClusterAlias,
8587
BiConsumer<ClusterState, Exception> handler) {
88+
assertThat(headers, sameInstance(autoFollowHeaders.get("remote")));
8689
handler.accept(leaderState, null);
8790
}
8891

@@ -91,6 +94,7 @@ void createAndFollow(Map<String, String> headers,
9194
FollowIndexAction.Request followRequest,
9295
Runnable successHandler,
9396
Consumer<Exception> failureHandler) {
97+
assertThat(headers, sameInstance(autoFollowHeaders.get("remote")));
9498
assertThat(followRequest.getLeaderIndex(), equalTo("remote:logs-20190101"));
9599
assertThat(followRequest.getFollowerIndex(), equalTo("logs-20190101"));
96100
successHandler.run();
@@ -115,12 +119,13 @@ public void testAutoFollowerClusterStateApiFailure() {
115119
when(client.getRemoteClusterClient(anyString())).thenReturn(client);
116120

117121
AutoFollowPattern autoFollowPattern =
118-
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null, null);
122+
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null);
119123
Map<String, AutoFollowPattern> patterns = new HashMap<>();
120124
patterns.put("remote", autoFollowPattern);
121125
Map<String, List<String>> followedLeaderIndexUUIDS = new HashMap<>();
122126
followedLeaderIndexUUIDS.put("remote", new ArrayList<>());
123-
AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(patterns, followedLeaderIndexUUIDS);
127+
Map<String, Map<String, String>> headers = new HashMap<>();
128+
AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(patterns, followedLeaderIndexUUIDS, headers);
124129
ClusterState followerState = ClusterState.builder(new ClusterName("remote"))
125130
.metaData(MetaData.builder().putCustom(AutoFollowMetadata.TYPE, autoFollowMetadata))
126131
.build();
@@ -172,12 +177,13 @@ public void testAutoFollowerUpdateClusterStateFailure() {
172177
.build();
173178

174179
AutoFollowPattern autoFollowPattern =
175-
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null, null);
180+
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null);
176181
Map<String, AutoFollowPattern> patterns = new HashMap<>();
177182
patterns.put("remote", autoFollowPattern);
178183
Map<String, List<String>> followedLeaderIndexUUIDS = new HashMap<>();
179184
followedLeaderIndexUUIDS.put("remote", new ArrayList<>());
180-
AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(patterns, followedLeaderIndexUUIDS);
185+
Map<String, Map<String, String>> headers = new HashMap<>();
186+
AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(patterns, followedLeaderIndexUUIDS, headers);
181187
ClusterState followerState = ClusterState.builder(new ClusterName("remote"))
182188
.metaData(MetaData.builder().putCustom(AutoFollowMetadata.TYPE, autoFollowMetadata))
183189
.build();
@@ -233,12 +239,13 @@ public void testAutoFollowerCreateAndFollowApiCallFailure() {
233239
.build();
234240

235241
AutoFollowPattern autoFollowPattern =
236-
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null, null);
242+
new AutoFollowPattern(Collections.singletonList("logs-*"), null, null, null, null, null, null, null, null);
237243
Map<String, AutoFollowPattern> patterns = new HashMap<>();
238244
patterns.put("remote", autoFollowPattern);
239245
Map<String, List<String>> followedLeaderIndexUUIDS = new HashMap<>();
240246
followedLeaderIndexUUIDS.put("remote", new ArrayList<>());
241-
AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(patterns, followedLeaderIndexUUIDS);
247+
Map<String, Map<String, String>> headers = new HashMap<>();
248+
AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(patterns, followedLeaderIndexUUIDS, headers);
242249
ClusterState followerState = ClusterState.builder(new ClusterName("remote"))
243250
.metaData(MetaData.builder().putCustom(AutoFollowMetadata.TYPE, autoFollowMetadata))
244251
.build();
@@ -285,10 +292,11 @@ void updateAutoFollowMetadata(Function<ClusterState, ClusterState> updateFunctio
285292

286293
public void testGetLeaderIndicesToFollow() {
287294
AutoFollowPattern autoFollowPattern =
288-
new AutoFollowPattern(Collections.singletonList("metrics-*"), null, null, null, null, null, null, null, null, null);
295+
new AutoFollowPattern(Collections.singletonList("metrics-*"), null, null, null, null, null, null, null, null);
296+
Map<String, Map<String, String>> headers = new HashMap<>();
289297
ClusterState followerState = ClusterState.builder(new ClusterName("remote"))
290298
.metaData(MetaData.builder().putCustom(AutoFollowMetadata.TYPE,
291-
new AutoFollowMetadata(Collections.singletonMap("remote", autoFollowPattern), Collections.emptyMap())))
299+
new AutoFollowMetadata(Collections.singletonMap("remote", autoFollowPattern), Collections.emptyMap(), headers)))
292300
.build();
293301

294302
MetaData.Builder imdBuilder = MetaData.builder();
@@ -331,15 +339,15 @@ public void testGetLeaderIndicesToFollow() {
331339

332340
public void testGetFollowerIndexName() {
333341
AutoFollowPattern autoFollowPattern = new AutoFollowPattern(Collections.singletonList("metrics-*"), null, null,
334-
null, null, null, null, null, null, null);
342+
null, null, null, null, null, null);
335343
assertThat(AutoFollower.getFollowerIndexName(autoFollowPattern, "metrics-0"), equalTo("metrics-0"));
336344

337345
autoFollowPattern = new AutoFollowPattern(Collections.singletonList("metrics-*"), "eu-metrics-0", null, null,
338-
null, null, null, null, null, null);
346+
null, null, null, null, null);
339347
assertThat(AutoFollower.getFollowerIndexName(autoFollowPattern, "metrics-0"), equalTo("eu-metrics-0"));
340348

341349
autoFollowPattern = new AutoFollowPattern(Collections.singletonList("metrics-*"), "eu-{{leader_index}}", null,
342-
null, null, null, null, null, null, null);
350+
null, null, null, null, null, null);
343351
assertThat(AutoFollower.getFollowerIndexName(autoFollowPattern, "metrics-0"), equalTo("eu-metrics-0"));
344352
}
345353

0 commit comments

Comments
 (0)