Skip to content

Commit 0250d55

Browse files
jasontedorjasontedor
committed
Introduce forget follower API (#39718)
This commit introduces the forget follower API. This API is needed in cases that unfollowing a following index fails to remove the shard history retention leases on the leader index. This can happen explicitly through user action, or implicitly through an index managed by ILM. When this occurs, history will be retained longer than necessary. While the retention lease will eventually expire, it can be expensive to allow history to persist for that long, and also prevent ILM from performing actions like shrink on the leader index. As such, we introduce an API to allow for manual removal of the shard history retention leases in this case.
1 parent 6c75a2f commit 0250d55

File tree

29 files changed

+1448
-13
lines changed

29 files changed

+1448
-13
lines changed

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
104104
* format of the response is incompatible i.e. it is not a JSON object.
105105
*/
106106
static shouldAddShardFailureCheck(String path) {
107-
return path.startsWith('_cat') == false && path.startsWith('_ml/datafeeds/') == false
107+
return path.startsWith('_cat') == false && path.startsWith('_ml/datafeeds/') == false
108108
}
109109

110110
/**
@@ -294,7 +294,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
294294
}
295295

296296
void emitDo(String method, String pathAndQuery, String body,
297-
String catchPart, List warnings, boolean inSetup) {
297+
String catchPart, List warnings, boolean inSetup, boolean skipShardFailures) {
298298
def (String path, String query) = pathAndQuery.tokenize('?')
299299
if (path == null) {
300300
path = '' // Catch requests to the root...
@@ -346,7 +346,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
346346
* section so we have to skip it there. We also omit the assertion
347347
* from APIs that don't return a JSON object
348348
*/
349-
if (false == inSetup && shouldAddShardFailureCheck(path)) {
349+
if (false == inSetup && skipShardFailures == false && shouldAddShardFailureCheck(path)) {
350350
current.println(" - is_false: _shards.failures")
351351
}
352352
}
@@ -394,7 +394,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
394394
pathAndQuery = pathAndQuery.substring(1)
395395
}
396396
emitDo(method, pathAndQuery, body, catchPart, snippet.warnings,
397-
inSetup)
397+
inSetup, snippet.skipShardsFailures)
398398
}
399399
}
400400

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public class SnippetsTask extends DefaultTask {
4545
private static final String WARNING = /warning:(.+)/
4646
private static final String CAT = /(_cat)/
4747
private static final String TEST_SYNTAX =
48-
/(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP|$WARNING) ?/
48+
/(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP|$WARNING|(skip_shard_failures)) ?/
4949

5050
/**
5151
* Action to take on each snippet. Called with a single parameter, an
@@ -233,6 +233,10 @@ public class SnippetsTask extends DefaultTask {
233233
snippet.warnings.add(it.group(7))
234234
return
235235
}
236+
if (it.group(8) != null) {
237+
snippet.skipShardsFailures = true
238+
return
239+
}
236240
throw new InvalidUserDataException(
237241
"Invalid test marker: $line")
238242
}
@@ -329,6 +333,7 @@ public class SnippetsTask extends DefaultTask {
329333
String setup = null
330334
boolean curl
331335
List warnings = new ArrayList()
336+
boolean skipShardsFailures = false
332337

333338
@Override
334339
public String toString() {
@@ -359,6 +364,9 @@ public class SnippetsTask extends DefaultTask {
359364
for (String warning in warnings) {
360365
result += "[warning:$warning]"
361366
}
367+
if (skipShardsFailures) {
368+
result += '[skip_shard_failures]'
369+
}
362370
}
363371
if (testResponse) {
364372
result += '// TESTRESPONSE'

‎client/rest-high-level/src/main/java/org/elasticsearch/client/CcrClient.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.client.ccr.FollowInfoResponse;
2828
import org.elasticsearch.client.ccr.FollowStatsRequest;
2929
import org.elasticsearch.client.ccr.FollowStatsResponse;
30+
import org.elasticsearch.client.ccr.ForgetFollowerRequest;
3031
import org.elasticsearch.client.ccr.GetAutoFollowPatternRequest;
3132
import org.elasticsearch.client.ccr.GetAutoFollowPatternResponse;
3233
import org.elasticsearch.client.ccr.PauseFollowRequest;
@@ -36,6 +37,7 @@
3637
import org.elasticsearch.client.ccr.ResumeFollowRequest;
3738
import org.elasticsearch.client.ccr.UnfollowRequest;
3839
import org.elasticsearch.client.core.AcknowledgedResponse;
40+
import org.elasticsearch.client.core.BroadcastResponse;
3941

4042
import java.io.IOException;
4143
import java.util.Collections;
@@ -233,6 +235,48 @@ public void unfollowAsync(UnfollowRequest request,
233235
);
234236
}
235237

238+
/**
239+
* Instructs an index acting as a leader index to forget the specified follower index.
240+
*
241+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-forget-follower.html">the docs</a> for more details
242+
* on the intended usage of this API.
243+
*
244+
* @param request the request
245+
* @param options the request options (e.g., headers), use {@link RequestOptions#DEFAULT} if the defaults are acceptable.
246+
* @return the response
247+
* @throws IOException if an I/O exception occurs while executing this request
248+
*/
249+
public BroadcastResponse forgetFollower(final ForgetFollowerRequest request, final RequestOptions options) throws IOException {
250+
return restHighLevelClient.performRequestAndParseEntity(
251+
request,
252+
CcrRequestConverters::forgetFollower,
253+
options,
254+
BroadcastResponse::fromXContent,
255+
Collections.emptySet());
256+
}
257+
258+
/**
259+
* Asynchronously instructs an index acting as a leader index to forget the specified follower index.
260+
*
261+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-forget-follower.html">the docs</a> for more details
262+
* on the intended usage of this API.
263+
*
264+
* @param request the request
265+
* @param options the request options (e.g., headers), use {@link RequestOptions#DEFAULT} if the defaults are acceptable.
266+
*/
267+
public void forgetFollowerAsync(
268+
final ForgetFollowerRequest request,
269+
final RequestOptions options,
270+
final ActionListener<BroadcastResponse> listener) {
271+
restHighLevelClient.performRequestAsyncAndParseEntity(
272+
request,
273+
CcrRequestConverters::forgetFollower,
274+
options,
275+
BroadcastResponse::fromXContent,
276+
listener,
277+
Collections.emptySet());
278+
}
279+
236280
/**
237281
* Stores an auto follow pattern.
238282
*

‎client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.client.ccr.DeleteAutoFollowPatternRequest;
2828
import org.elasticsearch.client.ccr.FollowInfoRequest;
2929
import org.elasticsearch.client.ccr.FollowStatsRequest;
30+
import org.elasticsearch.client.ccr.ForgetFollowerRequest;
3031
import org.elasticsearch.client.ccr.GetAutoFollowPatternRequest;
3132
import org.elasticsearch.client.ccr.PauseFollowRequest;
3233
import org.elasticsearch.client.ccr.PutAutoFollowPatternRequest;
@@ -79,6 +80,17 @@ static Request unfollow(UnfollowRequest unfollowRequest) {
7980
return new Request(HttpPost.METHOD_NAME, endpoint);
8081
}
8182

83+
static Request forgetFollower(final ForgetFollowerRequest forgetFollowerRequest) throws IOException {
84+
final String endpoint = new RequestConverters.EndpointBuilder()
85+
.addPathPart(forgetFollowerRequest.leaderIndex())
86+
.addPathPartAsIs("_ccr")
87+
.addPathPartAsIs("forget_follower")
88+
.build();
89+
final Request request = new Request(HttpPost.METHOD_NAME, endpoint);
90+
request.setEntity(createEntity(forgetFollowerRequest, REQUEST_BODY_CONTENT_TYPE));
91+
return request;
92+
}
93+
8294
static Request putAutoFollowPattern(PutAutoFollowPatternRequest putAutoFollowPatternRequest) throws IOException {
8395
String endpoint = new RequestConverters.EndpointBuilder()
8496
.addPathPartAsIs("_ccr", "auto_follow")

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/ForgetFollowerRequest.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.client.ccr;
21+
22+
import org.elasticsearch.client.Validatable;
23+
import org.elasticsearch.common.xcontent.ToXContentObject;
24+
import org.elasticsearch.common.xcontent.XContentBuilder;
25+
26+
import java.io.IOException;
27+
import java.util.Objects;
28+
29+
/**
30+
* Represents a forget follower request. Note that this an expert API intended to be used only when unfollowing a follower index fails to
31+
* remove the follower retention leases. Please be sure that you understand the purpose this API before using.
32+
*/
33+
public final class ForgetFollowerRequest implements ToXContentObject, Validatable {
34+
35+
private final String followerCluster;
36+
37+
private final String followerIndex;
38+
39+
private final String followerIndexUUID;
40+
41+
private final String leaderRemoteCluster;
42+
43+
private final String leaderIndex;
44+
45+
/**
46+
* The name of the leader index.
47+
*
48+
* @return the name of the leader index
49+
*/
50+
public String leaderIndex() {
51+
return leaderIndex;
52+
}
53+
54+
/**
55+
* Construct a forget follower request.
56+
*
57+
* @param followerCluster the name of the cluster containing the follower index to forget
58+
* @param followerIndex the name of follower index
59+
* @param followerIndexUUID the UUID of the follower index
60+
* @param leaderRemoteCluster the alias of the remote cluster containing the leader index from the perspective of the follower index
61+
* @param leaderIndex the name of the leader index
62+
*/
63+
public ForgetFollowerRequest(
64+
final String followerCluster,
65+
final String followerIndex,
66+
final String followerIndexUUID,
67+
final String leaderRemoteCluster,
68+
final String leaderIndex) {
69+
this.followerCluster = Objects.requireNonNull(followerCluster);
70+
this.followerIndex = Objects.requireNonNull(followerIndex);
71+
this.followerIndexUUID = Objects.requireNonNull(followerIndexUUID);
72+
this.leaderRemoteCluster = Objects.requireNonNull(leaderRemoteCluster);
73+
this.leaderIndex = Objects.requireNonNull(leaderIndex);
74+
}
75+
76+
@Override
77+
public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
78+
builder.startObject();
79+
{
80+
builder.field("follower_cluster", followerCluster);
81+
builder.field("follower_index", followerIndex);
82+
builder.field("follower_index_uuid", followerIndexUUID);
83+
builder.field("leader_remote_cluster", leaderRemoteCluster);
84+
}
85+
builder.endObject();
86+
return builder;
87+
}
88+
89+
}

0 commit comments

Comments
 (0)