Skip to content

Commit 0193979

Browse files
talevytalevy
authored
Introduce Maintenance Mode to ILM (#31164)
This PR introduces a concept of a maintenance mode for the lifecycle service. During maintenance mode, no policies are executed. To be placed into maintenance mode, users must first issue a request to be placed in maintenance mode. Once the service is assured that no policies are in actions that are not to be interrupted (like ShrinkAction), the service will place itself in maintenance mode. APIs to-be introduced: - POST _xpack/index_lifecycle/maintenance/_request - issues a request to be placed into maintenenance mode. This is not immediate, since we must first verify that it is safe to go from REQUESTED -> IN maintenance mode. - POST _xpack/index_lifecycle/maintenance/_stop - issues a request to be taken out (this is immediate) - GET _xpack/index_lifecycle/maintenance - get back the current mode our lifecycle management is in
1 parent c549377 commit 0193979

File tree

14 files changed

+412
-244
lines changed

14 files changed

+412
-244
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleMetadata.java‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,24 +33,29 @@
3333

3434
public class IndexLifecycleMetadata implements XPackMetaDataCustom {
3535
public static final String TYPE = "index_lifecycle";
36+
public static final ParseField MAINTENANCE_MODE_FIELD = new ParseField("maintenance_mode");
3637
public static final ParseField POLICIES_FIELD = new ParseField("policies");
37-
public static final IndexLifecycleMetadata EMPTY = new IndexLifecycleMetadata(Collections.emptySortedMap());
38+
public static final IndexLifecycleMetadata EMPTY = new IndexLifecycleMetadata(Collections.emptySortedMap(), OperationMode.NORMAL);
3839

3940
@SuppressWarnings("unchecked")
4041
public static final ConstructingObjectParser<IndexLifecycleMetadata, Void> PARSER = new ConstructingObjectParser<>(
4142
TYPE, a -> new IndexLifecycleMetadata(
42-
ObjectParserUtils.convertListToMapValues(LifecyclePolicyMetadata::getName, (List<LifecyclePolicyMetadata>) a[0])));
43+
ObjectParserUtils.convertListToMapValues(LifecyclePolicyMetadata::getName, (List<LifecyclePolicyMetadata>) a[0]),
44+
OperationMode.valueOf((String) a[1])));
4345
static {
4446
PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> LifecyclePolicyMetadata.parse(p, n),
4547
v -> {
4648
throw new IllegalArgumentException("ordered " + POLICIES_FIELD.getPreferredName() + " are not supported");
4749
}, POLICIES_FIELD);
50+
PARSER.declareString(ConstructingObjectParser.constructorArg(), MAINTENANCE_MODE_FIELD);
4851
}
4952

5053
private final Map<String, LifecyclePolicyMetadata> policyMetadatas;
54+
private final OperationMode maintenanceMode;
5155

52-
public IndexLifecycleMetadata(Map<String, LifecyclePolicyMetadata> policies) {
56+
public IndexLifecycleMetadata(Map<String, LifecyclePolicyMetadata> policies, OperationMode maintenanceMode) {
5357
this.policyMetadatas = Collections.unmodifiableMap(policies);
58+
this.maintenanceMode = maintenanceMode;
5459
}
5560

5661
public IndexLifecycleMetadata(StreamInput in) throws IOException {
@@ -60,6 +65,7 @@ public IndexLifecycleMetadata(StreamInput in) throws IOException {
6065
policies.put(in.readString(), new LifecyclePolicyMetadata(in));
6166
}
6267
this.policyMetadatas = policies;
68+
this.maintenanceMode = in.readEnum(OperationMode.class);
6369
}
6470

6571
@Override
@@ -69,12 +75,17 @@ public void writeTo(StreamOutput out) throws IOException {
6975
out.writeString(entry.getKey());
7076
entry.getValue().writeTo(out);
7177
}
78+
out.writeEnum(maintenanceMode);
7279
}
7380

7481
public Map<String, LifecyclePolicyMetadata> getPolicyMetadatas() {
7582
return policyMetadatas;
7683
}
7784

85+
public OperationMode getMaintenanceMode() {
86+
return maintenanceMode;
87+
}
88+
7889
public Map<String, LifecyclePolicy> getPolicies() {
7990
return policyMetadatas.values().stream().map(LifecyclePolicyMetadata::getPolicy)
8091
.collect(Collectors.toMap(LifecyclePolicy::getName, Function.identity()));
@@ -88,6 +99,7 @@ public Diff<Custom> diff(Custom previousState) {
8899
@Override
89100
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
90101
builder.field(POLICIES_FIELD.getPreferredName(), policyMetadatas);
102+
builder.field(MAINTENANCE_MODE_FIELD.getPreferredName(), maintenanceMode);
91103
return builder;
92104
}
93105

@@ -108,7 +120,7 @@ public EnumSet<MetaData.XContentContext> context() {
108120

109121
@Override
110122
public int hashCode() {
111-
return Objects.hash(policyMetadatas);
123+
return Objects.hash(policyMetadatas, maintenanceMode);
112124
}
113125

114126
@Override
@@ -120,7 +132,8 @@ public boolean equals(Object obj) {
120132
return false;
121133
}
122134
IndexLifecycleMetadata other = (IndexLifecycleMetadata) obj;
123-
return Objects.equals(policyMetadatas, other.policyMetadatas);
135+
return Objects.equals(policyMetadatas, other.policyMetadatas)
136+
&& Objects.equals(maintenanceMode, other.maintenanceMode);
124137
}
125138

126139
@Override
@@ -131,26 +144,30 @@ public String toString() {
131144
public static class IndexLifecycleMetadataDiff implements NamedDiff<MetaData.Custom> {
132145

133146
final Diff<Map<String, LifecyclePolicyMetadata>> policies;
147+
final OperationMode maintenanceMode;
134148

135149
IndexLifecycleMetadataDiff(IndexLifecycleMetadata before, IndexLifecycleMetadata after) {
136150
this.policies = DiffableUtils.diff(before.policyMetadatas, after.policyMetadatas, DiffableUtils.getStringKeySerializer());
151+
this.maintenanceMode = after.maintenanceMode;
137152
}
138153

139154
public IndexLifecycleMetadataDiff(StreamInput in) throws IOException {
140155
this.policies = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), LifecyclePolicyMetadata::new,
141156
IndexLifecycleMetadataDiff::readLifecyclePolicyDiffFrom);
157+
this.maintenanceMode = in.readEnum(OperationMode.class);
142158
}
143159

144160
@Override
145161
public MetaData.Custom apply(MetaData.Custom part) {
146162
TreeMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(
147163
policies.apply(((IndexLifecycleMetadata) part).policyMetadatas));
148-
return new IndexLifecycleMetadata(newPolicies);
164+
return new IndexLifecycleMetadata(newPolicies, this.maintenanceMode);
149165
}
150166

151167
@Override
152168
public void writeTo(StreamOutput out) throws IOException {
153169
policies.writeTo(out);
170+
out.writeEnum(maintenanceMode);
154171
}
155172

156173
@Override

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414
public class LifecycleSettings {
1515
public static final String LIFECYCLE_POLL_INTERVAL = "indices.lifecycle.poll_interval";
16+
public static final String LIFECYCLE_MAINTENANCE_MODE = "indices.lifecycle.maintenance";
1617
public static final String LIFECYCLE_NAME = "index.lifecycle.name";
1718
public static final String LIFECYCLE_PHASE = "index.lifecycle.phase";
1819
public static final String LIFECYCLE_ACTION = "index.lifecycle.action";

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OperationMode.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.indexlifecycle;
7+
8+
/**
9+
* Enum representing the different modes that Index Lifecycle Service can operate in.
10+
*/
11+
public enum OperationMode {
12+
/**
13+
* This represents a state where no policies are executed
14+
*/
15+
MAINTENANCE {
16+
@Override
17+
public boolean isValidChange(OperationMode nextMode) {
18+
return nextMode == NORMAL;
19+
}
20+
},
21+
22+
/**
23+
* this representes a state where only sensitive actions (like {@link ShrinkAction}) will be executed
24+
* until they finish, at which point the operation mode will move to maintenance mode.
25+
*/
26+
MAINTENANCE_REQUESTED {
27+
@Override
28+
public boolean isValidChange(OperationMode nextMode) {
29+
return nextMode == NORMAL || nextMode == MAINTENANCE;
30+
}
31+
},
32+
33+
/**
34+
* Normal operation where all policies are executed as normal.
35+
*/
36+
NORMAL {
37+
@Override
38+
public boolean isValidChange(OperationMode nextMode) {
39+
return nextMode == MAINTENANCE_REQUESTED;
40+
}
41+
};
42+
43+
public abstract boolean isValidChange(OperationMode nextMode);
44+
}

‎x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ public void runPolicy(String policy, IndexMetaData indexMetaData, ClusterState c
5757
return;
5858
}
5959
Step currentStep = getCurrentStep(stepRegistry, policy, indexSettings);
60+
if (currentStep == null) {
61+
throw new IllegalStateException(
62+
"current step for index [" + indexMetaData.getIndex().getName() + "] with policy [" + policy + "] is not recognized");
63+
}
6064
logger.debug("running policy with current-step[" + currentStep.getKey() + "]");
6165
if (currentStep instanceof TerminalPolicyStep) {
6266
logger.debug("policy [" + policy + "] for index [" + indexMetaData.getIndex().getName() + "] complete, skipping execution");
@@ -303,7 +307,7 @@ public static ClusterState setPolicyForIndexes(final String newPolicyName, final
303307
return currentState;
304308
}
305309
}
306-
310+
307311
private static IndexMetaData.Builder setPolicyForIndex(final String newPolicyName, LifecyclePolicy newPolicy,
308312
List<String> failedIndexes,
309313
Index index, IndexMetaData indexMetadata) {
@@ -321,7 +325,7 @@ private static IndexMetaData.Builder setPolicyForIndex(final String newPolicyNam
321325
return null;
322326
}
323327
}
324-
328+
325329
private static boolean canSetPolicy(StepKey currentStepKey, String currentPolicyName, LifecyclePolicy newPolicy) {
326330
if (Strings.hasLength(currentPolicyName)) {
327331
if (ShrinkAction.NAME.equals(currentStepKey.getAction())) {

‎x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@
55
*/
66
package org.elasticsearch.xpack.indexlifecycle;
77

8+
import com.carrotsearch.hppc.cursors.ObjectCursor;
89
import org.apache.logging.log4j.Logger;
910
import org.apache.lucene.util.SetOnce;
1011
import org.elasticsearch.client.Client;
1112
import org.elasticsearch.cluster.ClusterChangedEvent;
1213
import org.elasticsearch.cluster.ClusterState;
1314
import org.elasticsearch.cluster.ClusterStateApplier;
1415
import org.elasticsearch.cluster.ClusterStateListener;
16+
import org.elasticsearch.cluster.metadata.IndexMetaData;
1517
import org.elasticsearch.cluster.service.ClusterService;
1618
import org.elasticsearch.common.Strings;
1719
import org.elasticsearch.common.component.AbstractComponent;
@@ -21,11 +23,15 @@
2123
import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
2224
import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
2325
import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
26+
import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
27+
import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
2428
import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
2529
import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
2630

2731
import java.io.Closeable;
2832
import java.time.Clock;
33+
import java.util.Collections;
34+
import java.util.Set;
2935
import java.util.function.LongSupplier;
3036

3137
/**
@@ -34,6 +40,7 @@
3440
public class IndexLifecycleService extends AbstractComponent
3541
implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable {
3642
private static final Logger logger = ESLoggerFactory.getLogger(IndexLifecycleService.class);
43+
private static final Set<String> IGNORE_ACTIONS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkAction.NAME);
3744

3845
private final SetOnce<SchedulerEngine> scheduler = new SetOnce<>();
3946
private final Clock clock;
@@ -93,7 +100,6 @@ public void clusterChanged(ClusterChangedEvent event) {
93100

94101
boolean pollIntervalSettingChanged = !pollInterval.equals(previousPollInterval);
95102

96-
97103
if (scheduler.get() == null) { // metadata installed and scheduler should be kicked off. start your engines.
98104
scheduler.set(new SchedulerEngine(clock));
99105
scheduler.get().register(this);
@@ -142,16 +148,51 @@ public void triggered(SchedulerEngine.Event event) {
142148
}
143149
}
144150

151+
/**
152+
* executes the policy execution on the appropriate indices by running cluster-state tasks per index.
153+
*
154+
* If maintenance-mode was requested, and it is safe to move into maintenance-mode, this will also be done here
155+
* when possible after no policies are executed.
156+
*
157+
* @param clusterState the current cluster state
158+
* @param fromClusterStateChange whether things are triggered from the cluster-state-listener or the scheduler
159+
*/
145160
void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange) {
161+
IndexLifecycleMetadata currentMetadata = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
162+
163+
if (currentMetadata == null) {
164+
return;
165+
}
166+
167+
OperationMode currentMode = currentMetadata.getMaintenanceMode();
168+
169+
if (OperationMode.MAINTENANCE.equals(currentMode)) {
170+
return;
171+
}
172+
173+
boolean safeToEnterMaintenanceMode = true; // true until proven false by a run policy
174+
146175
// loop through all indices in cluster state and filter for ones that are
147176
// managed by the Index Lifecycle Service they have a index.lifecycle.name setting
148177
// associated to a policy
149-
clusterState.metaData().indices().valuesIt().forEachRemaining((idxMeta) -> {
178+
for (ObjectCursor<IndexMetaData> cursor : clusterState.metaData().indices().values()) {
179+
IndexMetaData idxMeta = cursor.value;
150180
String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings());
151181
if (Strings.isNullOrEmpty(policyName) == false) {
182+
StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(idxMeta.getSettings());
183+
if (OperationMode.MAINTENANCE_REQUESTED == currentMode && stepKey != null
184+
&& IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction()) == false) {
185+
logger.info("skipping policy [" + policyName + "] for index [" + idxMeta.getIndex().getName()
186+
+ "]. maintenance mode requested");
187+
continue;
188+
}
152189
lifecycleRunner.runPolicy(policyName, idxMeta, clusterState, fromClusterStateChange);
190+
safeToEnterMaintenanceMode = false; // proven false!
153191
}
154-
});
192+
}
193+
if (safeToEnterMaintenanceMode && OperationMode.MAINTENANCE_REQUESTED == currentMode) {
194+
submitMaintenanceModeUpdate(OperationMode.MAINTENANCE);
195+
}
155196
}
156197

157198
@Override
@@ -161,4 +202,9 @@ public void close() {
161202
engine.stop();
162203
}
163204
}
205+
206+
public void submitMaintenanceModeUpdate(OperationMode mode) {
207+
clusterService.submitStateUpdateTask("ilm_maintenance_update",
208+
new MaintenanceModeUpdateTask(mode));
209+
}
164210
}

‎x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/MaintenanceModeUpdateTask.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.indexlifecycle;
7+
8+
import org.apache.logging.log4j.Logger;
9+
import org.elasticsearch.cluster.ClusterState;
10+
import org.elasticsearch.cluster.ClusterStateUpdateTask;
11+
import org.elasticsearch.cluster.metadata.MetaData;
12+
import org.elasticsearch.common.logging.ESLoggerFactory;
13+
import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
14+
import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
15+
16+
public class MaintenanceModeUpdateTask extends ClusterStateUpdateTask {
17+
private static final Logger logger = ESLoggerFactory.getLogger(MaintenanceModeUpdateTask.class);
18+
private final OperationMode mode;
19+
20+
public MaintenanceModeUpdateTask(OperationMode mode) {
21+
this.mode = mode;
22+
}
23+
24+
OperationMode getOperationMode() {
25+
return mode;
26+
}
27+
28+
@Override
29+
public ClusterState execute(ClusterState currentState) {
30+
IndexLifecycleMetadata currentMetadata = currentState.metaData().custom(IndexLifecycleMetadata.TYPE);
31+
32+
33+
if (currentMetadata.getMaintenanceMode().isValidChange(mode) == false) {
34+
return currentState;
35+
}
36+
37+
ClusterState.Builder builder = new ClusterState.Builder(currentState);
38+
MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
39+
metadataBuilder.putCustom(IndexLifecycleMetadata.TYPE,
40+
new IndexLifecycleMetadata(currentMetadata.getPolicyMetadatas(), mode));
41+
builder.metaData(metadataBuilder.build());
42+
return builder.build();
43+
}
44+
45+
@Override
46+
public void onFailure(String source, Exception e) {
47+
logger.error("unable to update lifecycle metadata with new mode [" + mode + "]", e);
48+
}
49+
}

‎x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ public ClusterState execute(ClusterState currentState) {
8181
}
8282
SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
8383
newPolicies.remove(request.getPolicyName());
84-
IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies);
84+
IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentMetadata.getMaintenanceMode());
8585
newState.metaData(MetaData.builder(currentState.getMetaData())
8686
.putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
8787
return newState.build();

‎x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.xpack.core.ClientHelper;
2424
import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
2525
import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
26+
import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
2627
import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
2728
import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Request;
2829
import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Response;
@@ -82,7 +83,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
8283
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
8384
LifecyclePolicyMetadata lifecyclePolicyMetadata = new LifecyclePolicyMetadata(request.getPolicy(), filteredHeaders);
8485
newPolicies.put(lifecyclePolicyMetadata.getName(), lifecyclePolicyMetadata);
85-
IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies);
86+
IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, OperationMode.NORMAL);
8687
newState.metaData(MetaData.builder(currentState.getMetaData())
8788
.putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
8889
return newState.build();

0 commit comments

Comments
 (0)