Skip to content

Commit 7c738fd

Browse files
talevytalevy
authored
Skip Shrink when numberOfShards not changed (#37953)
Previously, ShrinkAction would fail if it was executed on an index that had the same number of shards as the target shrunken number. This PR introduced a new BranchingStep that is used inside of ShrinkAction to branch which step to move to next, depending on the shard values. So no shrink will occur if the shard count is unchanged.
1 parent b88bdfe commit 7c738fd

File tree

9 files changed

+346
-38
lines changed

9 files changed

+346
-38
lines changed

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -591,7 +591,7 @@ public void testRetryPolicy() throws Exception {
591591
{
592592
Map<String, Phase> phases = new HashMap<>();
593593
Map<String, LifecycleAction> warmActions = new HashMap<>();
594-
warmActions.put(ShrinkAction.NAME, new ShrinkAction(1));
594+
warmActions.put(ShrinkAction.NAME, new ShrinkAction(3));
595595
phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions));
596596

597597
LifecyclePolicy policy = new LifecyclePolicy("my_policy",
@@ -602,7 +602,7 @@ public void testRetryPolicy() throws Exception {
602602

603603
CreateIndexRequest createIndexRequest = new CreateIndexRequest("my_index")
604604
.settings(Settings.builder()
605-
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
605+
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
606606
.put("index.lifecycle.name", "my_policy")
607607
.build());
608608
client.indices().create(createIndexRequest, RequestOptions.DEFAULT);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStep.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.core.indexlifecycle;
8+
9+
import org.apache.logging.log4j.LogManager;
10+
import org.apache.logging.log4j.Logger;
11+
import org.apache.lucene.util.SetOnce;
12+
import org.elasticsearch.cluster.ClusterState;
13+
import org.elasticsearch.cluster.metadata.IndexMetaData;
14+
import org.elasticsearch.index.Index;
15+
16+
import java.util.Objects;
17+
import java.util.function.BiPredicate;
18+
19+
/**
20+
* This step changes its {@link #getNextStepKey()} depending on the
21+
* outcome of a defined predicate. It performs no changes to the
22+
* cluster state.
23+
*/
24+
public class BranchingStep extends ClusterStateActionStep {
25+
public static final String NAME = "branch";
26+
27+
private static final Logger logger = LogManager.getLogger(BranchingStep.class);
28+
29+
private StepKey nextStepKeyOnFalse;
30+
private StepKey nextStepKeyOnTrue;
31+
private BiPredicate<Index, ClusterState> predicate;
32+
private SetOnce<Boolean> predicateValue;
33+
34+
/**
35+
* {@link BranchingStep} is a step whose next step is based on
36+
* the return value of a specific predicate.
37+
*
38+
* @param key the step's key
39+
* @param nextStepKeyOnFalse the key of the step to run if predicate returns false
40+
* @param nextStepKeyOnTrue the key of the step to run if predicate returns true
41+
* @param predicate the condition to check when deciding which step to run next
42+
*/
43+
public BranchingStep(StepKey key, StepKey nextStepKeyOnFalse, StepKey nextStepKeyOnTrue, BiPredicate<Index, ClusterState> predicate) {
44+
// super.nextStepKey is set to null since it is not used by this step
45+
super(key, null);
46+
this.nextStepKeyOnFalse = nextStepKeyOnFalse;
47+
this.nextStepKeyOnTrue = nextStepKeyOnTrue;
48+
this.predicate = predicate;
49+
this.predicateValue = new SetOnce<>();
50+
}
51+
52+
@Override
53+
public ClusterState performAction(Index index, ClusterState clusterState) {
54+
IndexMetaData indexMetaData = clusterState.metaData().index(index);
55+
if (indexMetaData == null) {
56+
// Index must have been since deleted, ignore it
57+
logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
58+
return clusterState;
59+
}
60+
predicateValue.set(predicate.test(index, clusterState));
61+
return clusterState;
62+
}
63+
64+
/**
65+
* This method returns the next step to execute based on the predicate. If
66+
* the predicate returned true, then nextStepKeyOnTrue is the key of the
67+
* next step to run, otherwise nextStepKeyOnFalse is.
68+
*
69+
* throws {@link UnsupportedOperationException} if performAction was not called yet
70+
*
71+
* @return next step to execute
72+
*/
73+
@Override
74+
public final StepKey getNextStepKey() {
75+
if (predicateValue.get() == null) {
76+
throw new IllegalStateException("Cannot call getNextStepKey before performAction");
77+
}
78+
return predicateValue.get() ? nextStepKeyOnTrue : nextStepKeyOnFalse;
79+
}
80+
81+
/**
82+
* @return the next step if {@code predicate} is false
83+
*/
84+
final StepKey getNextStepKeyOnFalse() {
85+
return nextStepKeyOnFalse;
86+
}
87+
88+
/**
89+
* @return the next step if {@code predicate} is true
90+
*/
91+
final StepKey getNextStepKeyOnTrue() {
92+
return nextStepKeyOnTrue;
93+
}
94+
95+
public final BiPredicate<Index, ClusterState> getPredicate() {
96+
return predicate;
97+
}
98+
99+
@Override
100+
public boolean equals(Object o) {
101+
if (this == o) return true;
102+
if (o == null || getClass() != o.getClass()) return false;
103+
if (!super.equals(o)) return false;
104+
BranchingStep that = (BranchingStep) o;
105+
return super.equals(o)
106+
&& Objects.equals(nextStepKeyOnFalse, that.nextStepKeyOnFalse)
107+
&& Objects.equals(nextStepKeyOnTrue, that.nextStepKeyOnTrue);
108+
}
109+
110+
@Override
111+
public int hashCode() {
112+
return Objects.hash(super.hashCode(), nextStepKeyOnFalse, nextStepKeyOnTrue);
113+
}
114+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ public boolean isSafeAction() {
8585
public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
8686
Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build();
8787

88+
StepKey branchingKey = new StepKey(phase, NAME, BranchingStep.NAME);
8889
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
8990
StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
9091
StepKey allocationRoutedKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
@@ -94,6 +95,8 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
9495
StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
9596
StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
9697

98+
BranchingStep conditionalSkipShrinkStep = new BranchingStep(branchingKey, readOnlyKey, nextStepKey,
99+
(index, clusterState) -> clusterState.getMetaData().index(index).getNumberOfShards() == numberOfShards);
97100
UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, setSingleNodeKey, client, readOnlySettings);
98101
SetSingleNodeAllocateStep setSingleNodeStep = new SetSingleNodeAllocateStep(setSingleNodeKey, allocationRoutedKey, client);
99102
CheckShrinkReadyStep checkShrinkReadyStep = new CheckShrinkReadyStep(allocationRoutedKey, shrinkKey);
@@ -102,12 +105,13 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
102105
CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep(copyMetadataKey, aliasKey, SHRUNKEN_INDEX_PREFIX);
103106
ShrinkSetAliasStep aliasSwapAndDelete = new ShrinkSetAliasStep(aliasKey, isShrunkIndexKey, client, SHRUNKEN_INDEX_PREFIX);
104107
ShrunkenIndexCheckStep waitOnShrinkTakeover = new ShrunkenIndexCheckStep(isShrunkIndexKey, nextStepKey, SHRUNKEN_INDEX_PREFIX);
105-
return Arrays.asList(readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated, copyMetadata,
106-
aliasSwapAndDelete, waitOnShrinkTakeover);
108+
return Arrays.asList(conditionalSkipShrinkStep, readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated,
109+
copyMetadata, aliasSwapAndDelete, waitOnShrinkTakeover);
107110
}
108111

109112
@Override
110113
public List<StepKey> toStepKeys(String phase) {
114+
StepKey conditionalSkipKey = new StepKey(phase, NAME, BranchingStep.NAME);
111115
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
112116
StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
113117
StepKey checkShrinkReadyKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
@@ -116,7 +120,7 @@ public List<StepKey> toStepKeys(String phase) {
116120
StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
117121
StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
118122
StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
119-
return Arrays.asList(readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey,
123+
return Arrays.asList(conditionalSkipKey, readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey,
120124
copyMetadataKey, aliasKey, isShrunkIndexKey);
121125
}
122126

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public final StepKey getKey() {
3434
return key;
3535
}
3636

37-
public final StepKey getNextStepKey() {
37+
public StepKey getNextStepKey() {
3838
return nextStepKey;
3939
}
4040

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStepTests.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.indexlifecycle;
7+
8+
import org.apache.lucene.util.SetOnce;
9+
import org.elasticsearch.Version;
10+
import org.elasticsearch.cluster.ClusterName;
11+
import org.elasticsearch.cluster.ClusterState;
12+
import org.elasticsearch.cluster.metadata.IndexMetaData;
13+
import org.elasticsearch.cluster.metadata.MetaData;
14+
import org.elasticsearch.index.Index;
15+
import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
16+
17+
import java.util.function.BiPredicate;
18+
19+
import static org.hamcrest.Matchers.equalTo;
20+
21+
public class BranchingStepTests extends AbstractStepTestCase<BranchingStep> {
22+
23+
public void testPredicateNextStepChange() {
24+
String indexName = randomAlphaOfLength(5);
25+
ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(MetaData.builder()
26+
.put(IndexMetaData.builder(indexName).settings(settings(Version.CURRENT))
27+
.numberOfShards(1).numberOfReplicas(0))).build();
28+
StepKey stepKey = new StepKey(randomAlphaOfLength(5), randomAlphaOfLength(5), BranchingStep.NAME);
29+
StepKey nextStepKey = new StepKey(randomAlphaOfLength(6), randomAlphaOfLength(6), BranchingStep.NAME);
30+
StepKey nextSkipKey = new StepKey(randomAlphaOfLength(7), randomAlphaOfLength(7), BranchingStep.NAME);
31+
{
32+
BranchingStep step = new BranchingStep(stepKey, nextStepKey, nextSkipKey, (i, c) -> true);
33+
expectThrows(IllegalStateException.class, step::getNextStepKey);
34+
step.performAction(state.metaData().index(indexName).getIndex(), state);
35+
assertThat(step.getNextStepKey(), equalTo(step.getNextStepKeyOnTrue()));
36+
expectThrows(SetOnce.AlreadySetException.class, () -> step.performAction(state.metaData().index(indexName).getIndex(), state));
37+
}
38+
{
39+
BranchingStep step = new BranchingStep(stepKey, nextStepKey, nextSkipKey, (i, c) -> false);
40+
expectThrows(IllegalStateException.class, step::getNextStepKey);
41+
step.performAction(state.metaData().index(indexName).getIndex(), state);
42+
assertThat(step.getNextStepKey(), equalTo(step.getNextStepKeyOnFalse()));
43+
expectThrows(SetOnce.AlreadySetException.class, () -> step.performAction(state.metaData().index(indexName).getIndex(), state));
44+
}
45+
}
46+
47+
@Override
48+
public BranchingStep createRandomInstance() {
49+
StepKey stepKey = new StepKey(randomAlphaOfLength(5), randomAlphaOfLength(5), BranchingStep.NAME);
50+
StepKey nextStepKey = new StepKey(randomAlphaOfLength(6), randomAlphaOfLength(6), BranchingStep.NAME);
51+
StepKey nextSkipKey = new StepKey(randomAlphaOfLength(7), randomAlphaOfLength(7), BranchingStep.NAME);
52+
return new BranchingStep(stepKey, nextStepKey, nextSkipKey, (i, c) -> randomBoolean());
53+
}
54+
55+
@Override
56+
public BranchingStep mutateInstance(BranchingStep instance) {
57+
StepKey key = instance.getKey();
58+
StepKey nextStepKey = instance.getNextStepKeyOnFalse();
59+
StepKey nextSkipStepKey = instance.getNextStepKeyOnTrue();
60+
BiPredicate<Index, ClusterState> predicate = instance.getPredicate();
61+
62+
switch (between(0, 2)) {
63+
case 0:
64+
key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
65+
break;
66+
case 1:
67+
nextStepKey = new StepKey(nextStepKey.getPhase(), nextStepKey.getAction(), nextStepKey.getName() + randomAlphaOfLength(5));
68+
break;
69+
case 2:
70+
nextSkipStepKey = new StepKey(nextSkipStepKey.getPhase(), nextSkipStepKey.getAction(),
71+
nextSkipStepKey.getName() + randomAlphaOfLength(5));
72+
break;
73+
default:
74+
throw new AssertionError("Illegal randomisation branch");
75+
}
76+
77+
return new BranchingStep(key, nextStepKey, nextSkipStepKey, predicate);
78+
}
79+
80+
@Override
81+
public BranchingStep copyInstance(BranchingStep instance) {
82+
return new BranchingStep(instance.getKey(), instance.getNextStepKeyOnFalse(), instance.getNextStepKeyOnTrue(),
83+
instance.getPredicate());
84+
}
85+
}

0 commit comments

Comments
 (0)