Skip to content

Commit 2e3eb4b

Browse files
dakronedakrone
authored
Add API to execute SLM retention on-demand (#47405) (#47463)
* Add API to execute SLM retention on-demand (#47405) This is a backport of #47405 This commit adds the `/_slm/_execute_retention` API endpoint. This endpoint kicks off SLM retention and then returns immediately. This in particular allows us to run retention without scheduling it (for entirely manual invocation) or perform a one-off cleanup. This commit also includes HLRC for the new API, and fixes an issue in SLMSnapshotBlockingIntegTests where retention invoked prior to the test completing could resurrect an index the internal test cluster cleanup had already deleted. Resolves #46508 Relates to #43663
1 parent 6acc5ca commit 2e3eb4b

File tree

14 files changed

+510
-84
lines changed

14 files changed

+510
-84
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleClient.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.elasticsearch.client.slm.DeleteSnapshotLifecyclePolicyRequest;
3838
import org.elasticsearch.client.slm.ExecuteSnapshotLifecyclePolicyRequest;
3939
import org.elasticsearch.client.slm.ExecuteSnapshotLifecyclePolicyResponse;
40+
import org.elasticsearch.client.slm.ExecuteSnapshotLifecycleRetentionRequest;
4041
import org.elasticsearch.client.slm.GetSnapshotLifecyclePolicyRequest;
4142
import org.elasticsearch.client.slm.GetSnapshotLifecyclePolicyResponse;
4243
import org.elasticsearch.client.slm.GetSnapshotLifecycleStatsRequest;
@@ -467,6 +468,44 @@ public Cancellable executeSnapshotLifecyclePolicyAsync(
467468
options, ExecuteSnapshotLifecyclePolicyResponse::fromXContent, listener, emptySet());
468469
}
469470

471+
/**
472+
* Execute snapshot lifecycle retention
473+
* See <pre>
474+
* https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/
475+
* java-rest-high-ilm-slm-execute-snapshot-lifecycle-retention.html
476+
* </pre>
477+
* for more.
478+
* @param request the request
479+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
480+
* @return the response
481+
* @throws IOException in case there is a problem sending the request or parsing back the response
482+
*/
483+
public AcknowledgedResponse executeSnapshotLifecycleRetention(ExecuteSnapshotLifecycleRetentionRequest request,
484+
RequestOptions options) throws IOException {
485+
return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::executeSnapshotLifecycleRetention,
486+
options, AcknowledgedResponse::fromXContent, emptySet());
487+
}
488+
489+
/**
490+
* Asynchronously execute snapshot lifecycle retention
491+
* See <pre>
492+
* https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/
493+
* java-rest-high-ilm-slm-execute-snapshot-lifecycle-retention.html
494+
* </pre>
495+
* for more.
496+
* @param request the request
497+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
498+
* @param listener the listener to be notified upon request completion
499+
* @return cancellable that may be used to cancel the request
500+
*/
501+
public Cancellable executeSnapshotLifecycleRetentionAsync(
502+
ExecuteSnapshotLifecycleRetentionRequest request, RequestOptions options,
503+
ActionListener<AcknowledgedResponse> listener) {
504+
return restHighLevelClient.performRequestAsyncAndParseEntity(
505+
request, IndexLifecycleRequestConverters::executeSnapshotLifecycleRetention,
506+
options, AcknowledgedResponse::fromXContent, listener, emptySet());
507+
}
508+
470509
/**
471510
* Retrieve snapshot lifecycle statistics.
472511
* See <pre>

‎client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleRequestConverters.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.elasticsearch.client.indexlifecycle.StopILMRequest;
3535
import org.elasticsearch.client.slm.DeleteSnapshotLifecyclePolicyRequest;
3636
import org.elasticsearch.client.slm.ExecuteSnapshotLifecyclePolicyRequest;
37+
import org.elasticsearch.client.slm.ExecuteSnapshotLifecycleRetentionRequest;
3738
import org.elasticsearch.client.slm.GetSnapshotLifecyclePolicyRequest;
3839
import org.elasticsearch.client.slm.GetSnapshotLifecycleStatsRequest;
3940
import org.elasticsearch.client.slm.PutSnapshotLifecyclePolicyRequest;
@@ -217,6 +218,18 @@ static Request executeSnapshotLifecyclePolicy(ExecuteSnapshotLifecyclePolicyRequ
217218
return request;
218219
}
219220

221+
static Request executeSnapshotLifecycleRetention(ExecuteSnapshotLifecycleRetentionRequest executeSnapshotLifecycleRetentionRequest) {
222+
Request request = new Request(HttpPost.METHOD_NAME,
223+
new RequestConverters.EndpointBuilder()
224+
.addPathPartAsIs("_slm/_execute_retention")
225+
.build());
226+
RequestConverters.Params params = new RequestConverters.Params();
227+
params.withMasterTimeout(executeSnapshotLifecycleRetentionRequest.masterNodeTimeout());
228+
params.withTimeout(executeSnapshotLifecycleRetentionRequest.timeout());
229+
request.addParameters(params.asMap());
230+
return request;
231+
}
232+
220233
static Request getSnapshotLifecycleStats(GetSnapshotLifecycleStatsRequest getSnapshotLifecycleStatsRequest) {
221234
String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_slm/stats").build();
222235
Request request = new Request(HttpGet.METHOD_NAME, endpoint);

‎client/rest-high-level/src/main/java/org/elasticsearch/client/slm/ExecuteSnapshotLifecycleRetentionRequest.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.client.slm;
21+
22+
import org.elasticsearch.client.TimedRequest;
23+
24+
public class ExecuteSnapshotLifecycleRetentionRequest extends TimedRequest {
25+
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import org.elasticsearch.client.slm.DeleteSnapshotLifecyclePolicyRequest;
5858
import org.elasticsearch.client.slm.ExecuteSnapshotLifecyclePolicyRequest;
5959
import org.elasticsearch.client.slm.ExecuteSnapshotLifecyclePolicyResponse;
60+
import org.elasticsearch.client.slm.ExecuteSnapshotLifecycleRetentionRequest;
6061
import org.elasticsearch.client.slm.GetSnapshotLifecyclePolicyRequest;
6162
import org.elasticsearch.client.slm.GetSnapshotLifecyclePolicyResponse;
6263
import org.elasticsearch.client.slm.GetSnapshotLifecycleStatsRequest;
@@ -987,6 +988,44 @@ public void onFailure(Exception e) {
987988
// end::slm-delete-snapshot-lifecycle-policy-execute-async
988989

989990
assertTrue(deleteResp.isAcknowledged());
991+
992+
//////// EXECUTE RETENTION
993+
// tag::slm-execute-snapshot-lifecycle-retention
994+
ExecuteSnapshotLifecycleRetentionRequest req =
995+
new ExecuteSnapshotLifecycleRetentionRequest();
996+
// end::slm-execute-snapshot-lifecycle-retention
997+
998+
// tag::slm-execute-snapshot-lifecycle-retention-execute
999+
AcknowledgedResponse retentionResp =
1000+
client.indexLifecycle()
1001+
.executeSnapshotLifecycleRetention(req,
1002+
RequestOptions.DEFAULT);
1003+
// end::slm-execute-snapshot-lifecycle-retention-execute
1004+
1005+
// tag::slm-execute-snapshot-lifecycle-retention-response
1006+
final boolean acked = retentionResp.isAcknowledged();
1007+
// end::slm-execute-snapshot-lifecycle-retention-response
1008+
1009+
// tag::slm-execute-snapshot-lifecycle-policy-execute-listener
1010+
ActionListener<AcknowledgedResponse> retentionListener =
1011+
new ActionListener<AcknowledgedResponse>() {
1012+
@Override
1013+
public void onResponse(AcknowledgedResponse r) {
1014+
assert r.isAcknowledged(); // <1>
1015+
}
1016+
1017+
@Override
1018+
public void onFailure(Exception e) {
1019+
// <2>
1020+
}
1021+
};
1022+
// end::slm-execute-snapshot-lifecycle-retention-execute-listener
1023+
1024+
// tag::slm-execute-snapshot-lifecycle-retention-execute-async
1025+
client.indexLifecycle()
1026+
.executeSnapshotLifecycleRetentionAsync(req,
1027+
RequestOptions.DEFAULT, retentionListener);
1028+
// end::slm-execute-snapshot-lifecycle-retention-execute-async
9901029
}
9911030

9921031
private void assertSnapshotExists(final RestHighLevelClient client, final String repo, final String snapshotName) throws Exception {

‎docs/java-rest/high-level/ilm/execute_snapshot_lifecycle_retention.asciidoc‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
--
2+
:api: slm-execute-snapshot-lifecycle-retention
3+
:request: ExecuteSnapshotLifecycleRetentionRequest
4+
:response: AcknowledgedResponse
5+
--
6+
[role="xpack"]
7+
[id="{upid}-{api}"]
8+
=== Execute Snapshot Lifecycle Retention API
9+
10+
11+
[id="{upid}-{api}-request"]
12+
==== Request
13+
14+
The Execute Snapshot Lifecycle Retention API allows you to execute Snapshot Lifecycle Management
15+
Retention immediately, rather than waiting for its regularly scheduled execution.
16+
17+
["source","java",subs="attributes,callouts,macros"]
18+
--------------------------------------------------
19+
include-tagged::{doc-tests-file}[{api}-request]
20+
--------------------------------------------------
21+
22+
[id="{upid}-{api}-response"]
23+
==== Response
24+
25+
The returned +{response}+ contains a boolean for whether the request was
26+
acknowledged by the master node.
27+
28+
["source","java",subs="attributes,callouts,macros"]
29+
--------------------------------------------------
30+
include-tagged::{doc-tests-file}[{api}-response]
31+
--------------------------------------------------
32+
33+
include::../execution.asciidoc[]
34+
35+

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/ExecuteSnapshotRetentionAction.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.core.slm.action;
8+
9+
import org.elasticsearch.action.ActionRequestValidationException;
10+
import org.elasticsearch.action.ActionType;
11+
import org.elasticsearch.action.support.master.AcknowledgedRequest;
12+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
13+
import org.elasticsearch.common.io.stream.StreamInput;
14+
import org.elasticsearch.common.xcontent.ToXContentObject;
15+
import org.elasticsearch.common.xcontent.XContentBuilder;
16+
17+
import java.io.IOException;
18+
19+
public class ExecuteSnapshotRetentionAction extends ActionType<AcknowledgedResponse> {
20+
public static final ExecuteSnapshotRetentionAction INSTANCE = new ExecuteSnapshotRetentionAction();
21+
public static final String NAME = "cluster:admin/slm/execute-retention";
22+
23+
protected ExecuteSnapshotRetentionAction() {
24+
super(NAME, AcknowledgedResponse::new);
25+
}
26+
27+
public static class Request extends AcknowledgedRequest<ExecuteSnapshotRetentionAction.Request> implements ToXContentObject {
28+
29+
public Request() { }
30+
31+
public Request(StreamInput in) throws IOException {
32+
super(in);
33+
}
34+
35+
@Override
36+
public ActionRequestValidationException validate() {
37+
return null;
38+
}
39+
40+
@Override
41+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
42+
builder.startObject();
43+
builder.endObject();
44+
return builder;
45+
}
46+
47+
@Override
48+
public int hashCode() {
49+
return super.hashCode();
50+
}
51+
52+
@Override
53+
public boolean equals(Object obj) {
54+
if (obj == null) {
55+
return false;
56+
}
57+
if (obj.getClass() != getClass()) {
58+
return false;
59+
}
60+
return true;
61+
}
62+
}
63+
}

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
6666
import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction;
6767
import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction;
68+
import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotRetentionAction;
6869
import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleAction;
6970
import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleStatsAction;
7071
import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
@@ -96,11 +97,13 @@
9697
import org.elasticsearch.xpack.slm.SnapshotRetentionTask;
9798
import org.elasticsearch.xpack.slm.action.RestDeleteSnapshotLifecycleAction;
9899
import org.elasticsearch.xpack.slm.action.RestExecuteSnapshotLifecycleAction;
100+
import org.elasticsearch.xpack.slm.action.RestExecuteSnapshotRetentionAction;
99101
import org.elasticsearch.xpack.slm.action.RestGetSnapshotLifecycleAction;
100102
import org.elasticsearch.xpack.slm.action.RestGetSnapshotLifecycleStatsAction;
101103
import org.elasticsearch.xpack.slm.action.RestPutSnapshotLifecycleAction;
102104
import org.elasticsearch.xpack.slm.action.TransportDeleteSnapshotLifecycleAction;
103105
import org.elasticsearch.xpack.slm.action.TransportExecuteSnapshotLifecycleAction;
106+
import org.elasticsearch.xpack.slm.action.TransportExecuteSnapshotRetentionAction;
104107
import org.elasticsearch.xpack.slm.action.TransportGetSnapshotLifecycleAction;
105108
import org.elasticsearch.xpack.slm.action.TransportGetSnapshotLifecycleStatsAction;
106109
import org.elasticsearch.xpack.slm.action.TransportPutSnapshotLifecycleAction;
@@ -247,7 +250,8 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
247250
new RestDeleteSnapshotLifecycleAction(restController),
248251
new RestGetSnapshotLifecycleAction(restController),
249252
new RestExecuteSnapshotLifecycleAction(restController),
250-
new RestGetSnapshotLifecycleStatsAction(restController)
253+
new RestGetSnapshotLifecycleStatsAction(restController),
254+
new RestExecuteSnapshotRetentionAction(restController)
251255
));
252256
}
253257
return handlers;
@@ -276,7 +280,8 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
276280
new ActionHandler<>(DeleteSnapshotLifecycleAction.INSTANCE, TransportDeleteSnapshotLifecycleAction.class),
277281
new ActionHandler<>(GetSnapshotLifecycleAction.INSTANCE, TransportGetSnapshotLifecycleAction.class),
278282
new ActionHandler<>(ExecuteSnapshotLifecycleAction.INSTANCE, TransportExecuteSnapshotLifecycleAction.class),
279-
new ActionHandler<>(GetSnapshotLifecycleStatsAction.INSTANCE, TransportGetSnapshotLifecycleStatsAction.class)
283+
new ActionHandler<>(GetSnapshotLifecycleStatsAction.INSTANCE, TransportGetSnapshotLifecycleStatsAction.class),
284+
new ActionHandler<>(ExecuteSnapshotRetentionAction.INSTANCE, TransportExecuteSnapshotRetentionAction.class)
280285
));
281286
}
282287
return actions;

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionService.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,13 @@
3131
public class SnapshotRetentionService implements LocalNodeMasterListener, Closeable {
3232

3333
static final String SLM_RETENTION_JOB_ID = "slm-retention-job";
34+
static final String SLM_RETENTION_MANUAL_JOB_ID = "slm-execute-manual-retention-job";
3435

3536
private static final Logger logger = LogManager.getLogger(SnapshotRetentionService.class);
3637

3738
private final SchedulerEngine scheduler;
39+
private final SnapshotRetentionTask retentionTask;
40+
private final Clock clock;
3841

3942
private volatile String slmRetentionSchedule;
4043
private volatile boolean isMaster = false;
@@ -43,8 +46,10 @@ public SnapshotRetentionService(Settings settings,
4346
Supplier<SnapshotRetentionTask> taskSupplier,
4447
ClusterService clusterService,
4548
Clock clock) {
49+
this.clock = clock;
4650
this.scheduler = new SchedulerEngine(settings, clock);
47-
this.scheduler.register(taskSupplier.get());
51+
this.retentionTask = taskSupplier.get();
52+
this.scheduler.register(this.retentionTask);
4853
this.slmRetentionSchedule = LifecycleSettings.SLM_RETENTION_SCHEDULE_SETTING.get(settings);
4954
clusterService.addLocalNodeMasterListener(this);
5055
clusterService.getClusterSettings().addSettingsUpdateConsumer(LifecycleSettings.SLM_RETENTION_SCHEDULE_SETTING,
@@ -91,6 +96,16 @@ private void cancelRetentionJob() {
9196
this.scheduler.scheduledJobIds().forEach(this.scheduler::remove);
9297
}
9398

99+
/**
100+
* Manually trigger snapshot retention
101+
*/
102+
public void triggerRetention() {
103+
if (this.isMaster) {
104+
long now = clock.millis();
105+
this.retentionTask.triggered(new SchedulerEngine.Event(SLM_RETENTION_MANUAL_JOB_ID, now, now));
106+
}
107+
}
108+
94109
@Override
95110
public String executorName() {
96111
return ThreadPool.Names.SNAPSHOT;

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,10 @@ public SnapshotRetentionTask(Client client, ClusterService clusterService, LongS
8484

8585
@Override
8686
public void triggered(SchedulerEngine.Event event) {
87-
assert event.getJobName().equals(SnapshotRetentionService.SLM_RETENTION_JOB_ID) :
88-
"expected id to be " + SnapshotRetentionService.SLM_RETENTION_JOB_ID + " but it was " + event.getJobName();
87+
assert event.getJobName().equals(SnapshotRetentionService.SLM_RETENTION_JOB_ID) ||
88+
event.getJobName().equals(SnapshotRetentionService.SLM_RETENTION_MANUAL_JOB_ID):
89+
"expected id to be " + SnapshotRetentionService.SLM_RETENTION_JOB_ID + " or " +
90+
SnapshotRetentionService.SLM_RETENTION_MANUAL_JOB_ID + " but it was " + event.getJobName();
8991

9092
final ClusterState state = clusterService.state();
9193
if (SnapshotLifecycleService.ilmStoppedOrStopping(state)) {

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotRetentionAction.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.slm.action;
8+
9+
import org.elasticsearch.client.node.NodeClient;
10+
import org.elasticsearch.rest.BaseRestHandler;
11+
import org.elasticsearch.rest.RestController;
12+
import org.elasticsearch.rest.RestRequest;
13+
import org.elasticsearch.rest.action.RestToXContentListener;
14+
import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotRetentionAction;
15+
16+
public class RestExecuteSnapshotRetentionAction extends BaseRestHandler {
17+
18+
public RestExecuteSnapshotRetentionAction(RestController controller) {
19+
controller.registerHandler(RestRequest.Method.POST, "/_slm/_execute_retention", this);
20+
}
21+
22+
@Override
23+
public String getName() {
24+
return "slm_execute_retention";
25+
}
26+
27+
@Override
28+
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
29+
ExecuteSnapshotRetentionAction.Request req = new ExecuteSnapshotRetentionAction.Request();
30+
req.timeout(request.paramAsTime("timeout", req.timeout()));
31+
req.masterNodeTimeout(request.paramAsTime("master_timeout", req.masterNodeTimeout()));
32+
return channel -> client.execute(ExecuteSnapshotRetentionAction.INSTANCE, req, new RestToXContentListener<>(channel));
33+
}
34+
}

0 commit comments

Comments
 (0)