From 485be0c5bdbe33beebff053b3793780494315301 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 30 Aug 2018 14:31:41 -0700 Subject: [PATCH 1/2] add new phase definition setting used for retrieving phase to execute Since policies can be updated independent of execution plans for the current phase being executed, it would be nice to know what the phase that is executing looks like in JSON. This PR does just that, while also using that index setting to recontruct the phase steps to execute (for consistency) --- .../InitializePolicyContextStep.java | 3 +- .../core/indexlifecycle/LifecyclePolicy.java | 2 +- .../indexlifecycle/LifecycleSettings.java | 3 ++ .../indexlifecycle/TerminalPolicyStep.java | 3 +- .../xpack/indexlifecycle/IndexLifecycle.java | 3 +- .../indexlifecycle/IndexLifecycleRunner.java | 28 ++++++++-- .../indexlifecycle/IndexLifecycleService.java | 7 +-- .../indexlifecycle/PolicyStepsRegistry.java | 54 +++++++++++++++---- .../ExecuteStepsUpdateTaskTests.java | 25 ++++++--- .../IndexLifecycleInitialisationIT.java | 38 +++++++++++-- .../IndexLifecycleRunnerTests.java | 48 +++++++++++------ .../IndexLifecycleServiceTests.java | 2 +- .../MoveToErrorStepUpdateTaskTests.java | 11 ++++ .../MoveToNextStepUpdateTaskTests.java | 45 +++++++++++----- .../PolicyStepsRegistryTests.java | 21 ++++---- 15 files changed, 223 insertions(+), 70 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java index 157488bb5e3c8..d22ed2ef2e688 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java @@ -12,7 +12,8 @@ import org.elasticsearch.index.Index; public final class InitializePolicyContextStep extends ClusterStateActionStep { - public static final StepKey KEY = new StepKey("new", "init", "init"); + public static final String INITIALIZATION_PHASE = "new"; + public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init"); public InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) { super(key, nextStepKey); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicy.java index abf46df9a161b..208aee8831b8e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicy.java @@ -90,7 +90,7 @@ public LifecyclePolicy(StreamInput in) throws IOException { * a {@link Map} of {@link Phase}s which make up this * {@link LifecyclePolicy}. */ - LifecyclePolicy(LifecycleType type, String name, Map phases) { + public LifecyclePolicy(LifecycleType type, String name, Map phases) { this.name = name; this.phases = phases; this.type = type; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java index f865b970339d7..027904a31bf65 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java @@ -24,6 +24,7 @@ public class LifecycleSettings { public static final String LIFECYCLE_FAILED_STEP = "index.lifecycle.failed_step"; public static final String LIFECYCLE_STEP_INFO = "index.lifecycle.step_info"; public static final String LIFECYCLE_SKIP = "index.lifecycle.skip"; + public static final String LIFECYCLE_PHASE_DEFINITION = "index.lifecycle.phase_definition"; public static final Setting LIFECYCLE_POLL_INTERVAL_SETTING = Setting.positiveTimeSetting(LIFECYCLE_POLL_INTERVAL, TimeValue.timeValueMinutes(10), Setting.Property.Dynamic, Setting.Property.NodeScope); @@ -49,4 +50,6 @@ public class LifecycleSettings { Setting.Property.IndexScope, Setting.Property.NotCopyableOnResize, Setting.Property.InternalIndex); public static final Setting LIFECYCLE_SKIP_SETTING = Setting.boolSetting(LIFECYCLE_SKIP, false, Setting.Property.Dynamic, Setting.Property.IndexScope); + public static final Setting LIFECYCLE_PHASE_DEFINITION_SETTING = Setting.simpleString(LIFECYCLE_PHASE_DEFINITION, + Setting.Property.Dynamic, Setting.Property.IndexScope); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStep.java index dfff7cd760e43..4ba1b4fd83c60 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStep.java @@ -6,7 +6,8 @@ package org.elasticsearch.xpack.core.indexlifecycle; public class TerminalPolicyStep extends Step { - public static final StepKey KEY = new StepKey("completed", "completed", "completed"); + public static final String COMPLETED_PHASE = "completed"; + public static final StepKey KEY = new StepKey(COMPLETED_PHASE, "completed", "completed"); public static final TerminalPolicyStep INSTANCE = new TerminalPolicyStep(KEY, null); TerminalPolicyStep(StepKey key, StepKey nextStepKey) { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java index 99b86c52e63e5..974e9d5a0bdb2 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java @@ -134,6 +134,7 @@ public List> getSettings() { LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING, LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING, LifecycleSettings.LIFECYCLE_SKIP_SETTING, + LifecycleSettings.LIFECYCLE_PHASE_DEFINITION_SETTING, RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING); } @@ -146,7 +147,7 @@ public Collection createComponents(Client client, ClusterService cluster return emptyList(); } indexLifecycleInitialisationService - .set(new IndexLifecycleService(settings, client, clusterService, getClock(), System::currentTimeMillis)); + .set(new IndexLifecycleService(settings, client, clusterService, getClock(), System::currentTimeMillis, xContentRegistry)); return Collections.singletonList(indexLifecycleInitialisationService.get()); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java index 379b903ab8ceb..8dee489ffe4c9 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java @@ -26,8 +26,10 @@ import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateActionStep; import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep; import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep; +import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings; +import org.elasticsearch.xpack.core.indexlifecycle.Phase; import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; import org.elasticsearch.xpack.core.indexlifecycle.Step; import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; @@ -206,7 +208,9 @@ static ClusterState moveClusterStateToStep(String indexName, ClusterState curren static ClusterState moveClusterStateToNextStep(Index index, ClusterState clusterState, StepKey currentStep, StepKey nextStep, LongSupplier nowSupplier) { IndexMetaData idxMeta = clusterState.getMetaData().index(index); - Settings.Builder indexSettings = moveIndexSettingsToNextStep(idxMeta.getSettings(), currentStep, nextStep, nowSupplier); + IndexLifecycleMetadata ilmMeta = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE); + LifecyclePolicy policy = ilmMeta.getPolicies().get(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings())); + Settings.Builder indexSettings = moveIndexSettingsToNextStep(policy, idxMeta.getSettings(), currentStep, nextStep, nowSupplier); ClusterState.Builder newClusterStateBuilder = newClusterStateWithIndexSettings(index, clusterState, indexSettings); return newClusterStateBuilder.build(); } @@ -214,11 +218,13 @@ static ClusterState moveClusterStateToNextStep(Index index, ClusterState cluster static ClusterState moveClusterStateToErrorStep(Index index, ClusterState clusterState, StepKey currentStep, Exception cause, LongSupplier nowSupplier) throws IOException { IndexMetaData idxMeta = clusterState.getMetaData().index(index); + IndexLifecycleMetadata ilmMeta = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE); + LifecyclePolicy policy = ilmMeta.getPolicies().get(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings())); XContentBuilder causeXContentBuilder = JsonXContent.contentBuilder(); causeXContentBuilder.startObject(); ElasticsearchException.generateThrowableXContent(causeXContentBuilder, ToXContent.EMPTY_PARAMS, cause); causeXContentBuilder.endObject(); - Settings.Builder indexSettings = moveIndexSettingsToNextStep(idxMeta.getSettings(), currentStep, + Settings.Builder indexSettings = moveIndexSettingsToNextStep(policy, idxMeta.getSettings(), currentStep, new StepKey(currentStep.getPhase(), currentStep.getAction(), ErrorStep.NAME), nowSupplier) .put(LifecycleSettings.LIFECYCLE_FAILED_STEP, currentStep.getName()) .put(LifecycleSettings.LIFECYCLE_STEP_INFO, BytesReference.bytes(causeXContentBuilder).utf8ToString()); @@ -247,8 +253,8 @@ ClusterState moveClusterStateToFailedStep(ClusterState currentState, String[] in return newState; } - private static Settings.Builder moveIndexSettingsToNextStep(Settings existingSettings, StepKey currentStep, StepKey nextStep, - LongSupplier nowSupplier) { + private static Settings.Builder moveIndexSettingsToNextStep(LifecyclePolicy policy, Settings existingSettings, + StepKey currentStep, StepKey nextStep, LongSupplier nowSupplier) { long nowAsMillis = nowSupplier.getAsLong(); Settings.Builder newSettings = Settings.builder().put(existingSettings).put(LifecycleSettings.LIFECYCLE_PHASE, nextStep.getPhase()) .put(LifecycleSettings.LIFECYCLE_ACTION, nextStep.getAction()).put(LifecycleSettings.LIFECYCLE_STEP, nextStep.getName()) @@ -257,6 +263,18 @@ private static Settings.Builder moveIndexSettingsToNextStep(Settings existingSet .put(LifecycleSettings.LIFECYCLE_FAILED_STEP, (String) null) .put(LifecycleSettings.LIFECYCLE_STEP_INFO, (String) null); if (currentStep.getPhase().equals(nextStep.getPhase()) == false) { + final String newPhaseDefinition; + if ("new".equals(nextStep.getPhase()) || TerminalPolicyStep.KEY.equals(nextStep)) { + newPhaseDefinition = nextStep.getPhase(); + } else { + Phase nextPhase = policy.getPhases().get(nextStep.getPhase()); + if (nextPhase == null) { + newPhaseDefinition = null; + } else { + newPhaseDefinition = Strings.toString(nextPhase, false, false); + } + } + newSettings.put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, newPhaseDefinition); newSettings.put(LifecycleSettings.LIFECYCLE_PHASE_TIME, nowAsMillis); } if (currentStep.getAction().equals(nextStep.getAction()) == false) { @@ -356,7 +374,7 @@ private static IndexMetaData.Builder setPolicyForIndex(final String newPolicyNam // next available step StepKey nextValidStepKey = newPolicy.getNextValidStep(currentStepKey); if (nextValidStepKey.equals(currentStepKey) == false) { - newSettings = moveIndexSettingsToNextStep(idxSettings, currentStepKey, nextValidStepKey, nowSupplier); + newSettings = moveIndexSettingsToNextStep(newPolicy, idxSettings, currentStepKey, nextValidStepKey, nowSupplier); } } newSettings.put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), newPolicyName); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java index 561dcafcf2a25..1ebd7afa8c6cd 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.indexlifecycle.OperationMode; import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata; @@ -56,14 +57,15 @@ public class IndexLifecycleService extends AbstractComponent private LongSupplier nowSupplier; private SchedulerEngine.Job scheduledJob; - public IndexLifecycleService(Settings settings, Client client, ClusterService clusterService, Clock clock, LongSupplier nowSupplier) { + public IndexLifecycleService(Settings settings, Client client, ClusterService clusterService, Clock clock, LongSupplier nowSupplier, + NamedXContentRegistry xContentRegistry) { super(settings); this.client = client; this.clusterService = clusterService; this.clock = clock; this.nowSupplier = nowSupplier; this.scheduledJob = null; - this.policyRegistry = new PolicyStepsRegistry(); + this.policyRegistry = new PolicyStepsRegistry(xContentRegistry); this.lifecycleRunner = new IndexLifecycleRunner(policyRegistry, clusterService, nowSupplier); this.pollInterval = LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING.get(settings); clusterService.addStateApplier(this); @@ -143,7 +145,6 @@ public void applyClusterState(ClusterChangedEvent event) { policyRegistry.removeIndices(event.indicesDeleted()); } if (event.state().metaData().custom(IndexLifecycleMetadata.TYPE) != null) { - // update policy steps registry policyRegistry.update(event.state(), client, nowSupplier); } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java index 2715b9ae09bc1..6446896740d40 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java @@ -16,14 +16,23 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.Index; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep; import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.indexlifecycle.InitializePolicyContextStep; +import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy; import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings; +import org.elasticsearch.xpack.core.indexlifecycle.Phase; import org.elasticsearch.xpack.core.indexlifecycle.Step; +import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -44,21 +53,24 @@ public class PolicyStepsRegistry { private final Map> stepMap; // A map of index to a list of compiled steps for the current phase private final Map> indexPhaseSteps; + private final NamedXContentRegistry xContentRegistry; - public PolicyStepsRegistry() { + public PolicyStepsRegistry(NamedXContentRegistry xContentRegistry) { this.lifecyclePolicyMap = new TreeMap<>(); this.firstStepMap = new HashMap<>(); this.stepMap = new HashMap<>(); this.indexPhaseSteps = new HashMap<>(); + this.xContentRegistry = xContentRegistry; } PolicyStepsRegistry(SortedMap lifecyclePolicyMap, Map firstStepMap, Map> stepMap, - Map> indexPhaseSteps) { + Map> indexPhaseSteps, NamedXContentRegistry xContentRegistry) { this.lifecyclePolicyMap = lifecyclePolicyMap; this.firstStepMap = firstStepMap; this.stepMap = stepMap; this.indexPhaseSteps = indexPhaseSteps; + this.xContentRegistry = xContentRegistry; } SortedMap getLifecyclePolicyMap() { @@ -138,7 +150,7 @@ public LifecyclePolicyMetadata read(StreamInput in, String key) { for (ObjectCursor imd : clusterState.metaData().getIndices().values()) { final Index index = imd.value.getIndex(); final String policy = imd.value.getSettings().get(LifecycleSettings.LIFECYCLE_NAME); - if (policy == null) { + if (policy == null || lifecyclePolicyMap.containsKey(policy) == false) { indexPhaseSteps.remove(index); } else { final List currentSteps = indexPhaseSteps.get(index); @@ -146,22 +158,46 @@ public LifecyclePolicyMetadata read(StreamInput in, String key) { final String existingPhase = (currentSteps == null || currentSteps.size() == 0) ? "_none_" : currentSteps.get(0).getKey().getPhase(); // Retrieve the current phase, defaulting to "new" if no phase is set - final String currentPhase = imd.value.getSettings().get(LifecycleSettings.LIFECYCLE_PHASE, "new"); + final String currentPhase = imd.value.getSettings().get(LifecycleSettings.LIFECYCLE_PHASE, + InitializePolicyContextStep.INITIALIZATION_PHASE); if (existingPhase.equals(currentPhase) == false) { logger.debug("index [{}] has transitioned phases [{} -> {}], rebuilding step list", index, existingPhase, currentPhase); - // Only rebuild the index's steps if the phase of the existing steps does not match our index's current phase - final Map steps = stepMap.get(policy); - + // parse existing phase steps from the phase definition in the index settings + String phaseDef = imd.value.getSettings().get(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, + InitializePolicyContextStep.INITIALIZATION_PHASE); + final Phase phase; + LifecyclePolicy currentPolicy = lifecyclePolicyMap.get(policy).getPolicy(); + final LifecyclePolicy policyToExecute; + if (InitializePolicyContextStep.INITIALIZATION_PHASE.equals(phaseDef) + || TerminalPolicyStep.COMPLETED_PHASE.equals(phaseDef)) { + // It is ok to re-use potentially modified policy here since we are in an initialization or completed phase + policyToExecute = currentPolicy; + } else { + // if the current phase definition describes an internal step/phase, do not parse + try (XContentParser parser = JsonXContent.jsonXContent.createParser(xContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, phaseDef)) { + phase = Phase.parse(parser, currentPhase); + } catch (IOException e) { + logger.error("failed to configure phase [" + currentPhase + "] for index [" + index.getName() + "]", e); + indexPhaseSteps.remove(index); + continue; + } + Map phaseMap = new HashMap<>(currentPolicy.getPhases()); + if (phase != null) { + phaseMap.put(currentPhase, phase); + } + policyToExecute = new LifecyclePolicy(currentPolicy.getType(), currentPolicy.getName(), phaseMap); + } + final List steps = policyToExecute.toSteps(client, nowSupplier); // Build a list of steps that correspond with the phase the index is currently in final List phaseSteps; if (steps == null) { phaseSteps = new ArrayList<>(); } else { - phaseSteps = steps.entrySet().stream() + phaseSteps = steps.stream() .filter(e -> e.getKey().getPhase().equals(currentPhase)) - .map(Map.Entry::getValue) .collect(Collectors.toList()); } indexPhaseSteps.put(index, phaseSteps); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java index 70bb5cdec9aa1..7653ffc7383cc 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java @@ -15,11 +15,14 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.index.Index; import org.elasticsearch.node.Node; +import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.OperationMode; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata; @@ -49,9 +52,9 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase { - private static final StepKey firstStepKey = new StepKey("phase_1", "action_1", "step_1"); - private static final StepKey secondStepKey = new StepKey("phase_1", "action_1", "step_2"); - private static final StepKey thirdStepKey = new StepKey("phase_1", "action_1", "step_3"); + private static final StepKey firstStepKey = new StepKey("first_phase", "action_1", "step_1"); + private static final StepKey secondStepKey = new StepKey("first_phase", "action_1", "step_2"); + private static final StepKey thirdStepKey = new StepKey("first_phase", "action_1", "step_3"); private static final StepKey invalidStepKey = new StepKey("invalid", "invalid", "invalid"); private ClusterState clusterState; private PolicyStepsRegistry policyStepsRegistry; @@ -68,7 +71,7 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase { private String indexName; @Before - public void prepareState() { + public void prepareState() throws IOException { client = Mockito.mock(Client.class); Mockito.when(client.settings()).thenReturn(Settings.EMPTY); firstStep = new MockClusterStateActionStep(firstStepKey, secondStepKey); @@ -96,7 +99,7 @@ public void prepareState() { policyMap.put(mixedPolicyName, new LifecyclePolicyMetadata(mixedPolicy, Collections.emptyMap())); policyMap.put(allClusterPolicyName, new LifecyclePolicyMetadata(allClusterPolicy, Collections.emptyMap())); policyMap.put(invalidPolicyName, new LifecyclePolicyMetadata(invalidPolicy, Collections.emptyMap())); - policyStepsRegistry = new PolicyStepsRegistry(); + policyStepsRegistry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY); indexName = randomAlphaOfLength(5); lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING); @@ -130,10 +133,16 @@ private void setupIndexPolicy(String policyName) { } public void testExecuteAllUntilEndOfPhase() throws IOException { + NamedXContentRegistry registry = new NamedXContentRegistry( + Collections.singletonList(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MockAction.NAME), + (p) -> { + MockAction.parse(p); + return new MockAction(Arrays.asList(firstStep, allClusterSecondStep)); + }))); + policyStepsRegistry = new PolicyStepsRegistry(registry); setupIndexPolicy(allClusterPolicyName); Step startStep = policyStepsRegistry.getFirstStep(allClusterPolicyName); - Step afterStep = policyStepsRegistry.getStep(index, startStep.getNextStepKey()); long now = randomNonNegativeLong(); // test execute start till end of phase `new` ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(allClusterPolicyName, index, startStep, policyStepsRegistry, () -> now); @@ -236,7 +245,7 @@ public void testExecuteIncompleteWaitStepWithInfo() throws IOException { equalTo(stepInfo.toString())); } - public void testOnFailure() { + public void testOnFailure() throws IOException { setStateToKey(secondStepKey); Step startStep = policyStepsRegistry.getStep(index, secondStepKey); long now = randomNonNegativeLong(); @@ -249,7 +258,7 @@ public void testOnFailure() { assertSame(expectedException, exception.getCause()); } - private void setStateToKey(StepKey stepKey) { + private void setStateToKey(StepKey stepKey) throws IOException { clusterState = ClusterState.builder(clusterState) .metaData(MetaData.builder(clusterState.metaData()) .put(IndexMetaData.builder(clusterState.metaData().index(indexName)) diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationIT.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationIT.java index 63463dd02c565..77d0bd84ce8f4 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationIT.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; @@ -18,6 +19,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.index.Index; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -61,6 +63,13 @@ public class IndexLifecycleInitialisationIT extends ESIntegTestCase { private Settings settings; private LifecyclePolicy lifecyclePolicy; + private static final ObservableAction OBSERVABLE_ACTION; + static { + List steps = new ArrayList<>(); + Step.StepKey key = new Step.StepKey("mock", ObservableAction.NAME, ObservableClusterStateWaitStep.NAME); + steps.add(new ObservableClusterStateWaitStep(key, TerminalPolicyStep.KEY)); + OBSERVABLE_ACTION = new ObservableAction(steps, true); + } @Override protected Settings nodeSettings(int nodeOrdinal) { @@ -111,7 +120,7 @@ public void init() { List steps = new ArrayList<>(); Step.StepKey key = new Step.StepKey("mock", ObservableAction.NAME, ObservableClusterStateWaitStep.NAME); steps.add(new ObservableClusterStateWaitStep(key, TerminalPolicyStep.KEY)); - Map actions = Collections.singletonMap(ObservableAction.NAME, new ObservableAction(steps, true)); + Map actions = Collections.singletonMap(ObservableAction.NAME, OBSERVABLE_ACTION); Map phases = Collections.singletonMap("mock", new Phase("mock", TimeValue.timeValueSeconds(0), actions)); lifecyclePolicy = newLockableLifecyclePolicy("test", phases); } @@ -230,6 +239,16 @@ public void testMasterFailover() throws Exception { assertThat(step, equalTo(ObservableClusterStateWaitStep.NAME)); }); + if (randomBoolean()) { + // this checks that the phase execution is picked up from the phase definition settings + logger.info("updating lifecycle [test_lifecycle] to be empty"); + PutLifecycleAction.Request updateLifecycleRequest = new PutLifecycleAction.Request + (newLockableLifecyclePolicy(lifecyclePolicy.getName(), Collections.emptyMap())); + PutLifecycleAction.Response updateLifecycleResponse = client() + .execute(PutLifecycleAction.INSTANCE, updateLifecycleRequest).get(); + assertAcked(updateLifecycleResponse); + } + logger.info("Closing server1"); // kill the first server @@ -265,8 +284,9 @@ public void testPollIntervalUpdate() { } // update the poll interval - TimeValue newPollInterval = TimeValue.timeValueHours(randomLongBetween(6, 10)); - Settings newIntervalSettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, newPollInterval).build(); + TimeValue newPollInterval = TimeValue.timeValueHours(randomLongBetween(6, 1000)); + Settings newIntervalSettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, + newPollInterval.getStringRep()).build(); assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(newIntervalSettings)); { TimeValueSchedule schedule = (TimeValueSchedule) indexLifecycleService.getScheduledJob().getSchedule(); @@ -290,6 +310,18 @@ public List> getSettings() { Setting.Property.Dynamic, Setting.Property.IndexScope); return Collections.singletonList(COMPLETE_SETTING); } + + @Override + public List getNamedXContent() { + return Arrays.asList( + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ObservableAction.NAME), (p) -> { + MockAction.parse(p); + return OBSERVABLE_ACTION; + }) + ); + } + + @Override public List getNamedWriteables() { return Arrays.asList(new NamedWriteableRegistry.Entry(LifecycleType.class, LockableLifecycleType.TYPE, (in) -> LockableLifecycleType.INSTANCE), diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java index a9cc9eb3e2ffc..62bf921b44324 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java @@ -16,11 +16,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.Index; +import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests; import org.elasticsearch.xpack.core.indexlifecycle.OperationMode; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.indexlifecycle.AbstractStepTestCase; @@ -78,7 +80,7 @@ private PolicyStepsRegistry createOneStepPolicyStepRegistry(String policyName, S steps.add(step); Index index = new Index(indexName, indexName + "uuid"); indexSteps.put(index, steps); - return new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps); + return new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps, NamedXContentRegistry.EMPTY); } public void testRunPolicyTerminalPolicyStep() { @@ -340,7 +342,8 @@ public void testRunPolicyAsyncWaitStepClusterStateChangeIgnored() { public void testRunPolicyWithNoStepsInRegistry() { String policyName = "cluster_state_action_policy"; ClusterService clusterService = mock(ClusterService.class); - IndexLifecycleRunner runner = new IndexLifecycleRunner(new PolicyStepsRegistry(), clusterService, () -> 0L); + IndexLifecycleRunner runner = new IndexLifecycleRunner(new PolicyStepsRegistry(NamedXContentRegistry.EMPTY), + clusterService, () -> 0L); IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT)) .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); // verify that no exception is thrown @@ -466,7 +469,8 @@ public void testGetCurrentStep() { phase1Steps.add(thirdStep); Index index = new Index("test", "uuid"); indexSteps.put(index, phase1Steps); - PolicyStepsRegistry registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps); + PolicyStepsRegistry registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps, + NamedXContentRegistry.EMPTY); Settings indexSettings = Settings.EMPTY; Step actualStep = IndexLifecycleRunner.getCurrentStep(registry, policyName, index, indexSettings); @@ -500,7 +504,7 @@ public void testGetCurrentStep() { // TODO: it'd be nice if we used the actual registry.update method for this indexSteps.clear(); indexSteps.put(index, Collections.singletonList(fourthStep)); - registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps); + registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps, NamedXContentRegistry.EMPTY); indexSettings = Settings.builder() .put(LifecycleSettings.LIFECYCLE_PHASE, "phase_2") @@ -521,7 +525,7 @@ public void testGetCurrentStep() { // Back to phase_1 indexSteps.clear(); indexSteps.put(index, phase1Steps); - registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps); + registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps, NamedXContentRegistry.EMPTY); indexSettings = Settings.builder() .put(LifecycleSettings.LIFECYCLE_PHASE, "phase_1") @@ -550,24 +554,32 @@ public void testGetCurrentStep() { public void testMoveClusterStateToNextStep() { String indexName = "my_index"; + LifecyclePolicy policy = LifecyclePolicyTests.randomTestLifecyclePolicy("policy"); + Phase nextPhase = policy.getPhases().values().stream().findFirst().get(); + List policyMetadatas = Collections.singletonList( + new LifecyclePolicyMetadata(policy, Collections.emptyMap())); StepKey currentStep = new StepKey("current_phase", "current_action", "current_step"); - StepKey nextStep = new StepKey("next_phase", "next_action", "next_step"); + StepKey nextStep = new StepKey(nextPhase.getName(), "next_action", "next_step"); long now = randomNonNegativeLong(); - ClusterState clusterState = buildClusterState(indexName, Settings.builder(), Collections.emptyList()); + // test going from null lifecycle settings to next step + ClusterState clusterState = buildClusterState(indexName, + Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), policyMetadatas); Index index = clusterState.metaData().index(indexName).getIndex(); ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep, () -> now); assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now); - Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase()) - .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction()) - .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()); + // test going from set currentStep settings to nextStep + Builder indexSettingsBuilder = Settings.builder() + .put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()) + .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()); if (randomBoolean()) { indexSettingsBuilder.put(LifecycleSettings.LIFECYCLE_STEP_INFO, randomAlphaOfLength(20)); } - clusterState = buildClusterState(indexName, - indexSettingsBuilder, Collections.emptyList()); + clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas); index = clusterState.metaData().index(indexName).getIndex(); newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep, () -> now); assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now); @@ -626,8 +638,13 @@ public void testMoveClusterStateToNextStepSameAction() { public void testSuccessfulValidatedMoveClusterStateToNextStep() { String indexName = "my_index"; String policyName = "my_policy"; + LifecyclePolicy policy = randomValueOtherThanMany(p -> p.getPhases().size() == 0, + () -> LifecyclePolicyTests.randomTestLifecyclePolicy(policyName)); + Phase nextPhase = policy.getPhases().values().stream().findFirst().get(); + List policyMetadatas = Collections.singletonList( + new LifecyclePolicyMetadata(policy, Collections.emptyMap())); StepKey currentStepKey = new StepKey("current_phase", "current_action", "current_step"); - StepKey nextStepKey = new StepKey("next_phase", "next_action", "next_step"); + StepKey nextStepKey = new StepKey(nextPhase.getName(), "next_action", "next_step"); long now = randomNonNegativeLong(); Step step = new MockStep(nextStepKey, nextStepKey); PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step, indexName); @@ -636,7 +653,7 @@ public void testSuccessfulValidatedMoveClusterStateToNextStep() { .put(LifecycleSettings.LIFECYCLE_PHASE, currentStepKey.getPhase()) .put(LifecycleSettings.LIFECYCLE_ACTION, currentStepKey.getAction()) .put(LifecycleSettings.LIFECYCLE_STEP, currentStepKey.getName()); - ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.emptyList()); + ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas); Index index = clusterState.metaData().index(indexName).getIndex(); ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, currentStepKey, nextStepKey, () -> now, stepRegistry); @@ -866,7 +883,7 @@ public void testSetPolicyForIndex() { String newPolicyName = "new_policy"; String phaseName = randomAlphaOfLength(10); StepKey currentStep = new StepKey(phaseName, MockAction.NAME, randomAlphaOfLength(10)); - LifecyclePolicy newPolicy = createPolicy(oldPolicyName, + LifecyclePolicy newPolicy = createPolicy(newPolicyName, new StepKey(phaseName, MockAction.NAME, randomAlphaOfLength(9)), null); LifecyclePolicy oldPolicy = createPolicy(oldPolicyName, currentStep, null); Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName) @@ -875,6 +892,7 @@ public void testSetPolicyForIndex() { .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); List policyMetadatas = new ArrayList<>(); policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap())); + policyMetadatas.add(new LifecyclePolicyMetadata(newPolicy, Collections.emptyMap())); ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas); Index index = clusterState.metaData().index(indexName).getIndex(); Index[] indices = new Index[] { index }; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java index 73caecd01aeec..0444e14079192 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java @@ -94,7 +94,7 @@ public void prepareServices() { when(adminClient.indices()).thenReturn(indicesClient); when(client.settings()).thenReturn(Settings.EMPTY); - indexLifecycleService = new IndexLifecycleService(Settings.EMPTY, client, clusterService, clock, () -> now); + indexLifecycleService = new IndexLifecycleService(Settings.EMPTY, client, clusterService, clock, () -> now, null); Mockito.verify(clusterService).addListener(indexLifecycleService); Mockito.verify(clusterService).addStateApplier(indexLifecycleService); } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTaskTests.java index ab037647c579a..bbc31eb9aeeda 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTaskTests.java @@ -19,11 +19,17 @@ import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep; +import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy; +import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings; +import org.elasticsearch.xpack.core.indexlifecycle.OperationMode; import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; import org.junit.Before; import java.io.IOException; +import java.util.Collections; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.sameInstance; @@ -37,14 +43,19 @@ public class MoveToErrorStepUpdateTaskTests extends ESTestCase { @Before public void setupClusterState() { policy = randomAlphaOfLength(10); + LifecyclePolicy lifecyclePolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policy); IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5)) .settings(settings(Version.CURRENT) .put(LifecycleSettings.LIFECYCLE_NAME, policy)) .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); index = indexMetadata.getIndex(); + IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata( + Collections.singletonMap(policy, new LifecyclePolicyMetadata(lifecyclePolicy, Collections.emptyMap())), + OperationMode.RUNNING); MetaData metaData = MetaData.builder() .persistentSettings(settings(Version.CURRENT).build()) .put(IndexMetaData.builder(indexMetadata)) + .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta) .build(); clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build(); } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTaskTests.java index 677e049886aa1..8a74f01e6eeba 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTaskTests.java @@ -15,10 +15,19 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy; +import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings; +import org.elasticsearch.xpack.core.indexlifecycle.OperationMode; +import org.elasticsearch.xpack.core.indexlifecycle.Step; import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; import org.junit.Before; +import java.util.Collections; +import java.util.List; + import static org.hamcrest.Matchers.equalTo; public class MoveToNextStepUpdateTaskTests extends ESTestCase { @@ -26,6 +35,7 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase { String policy; ClusterState clusterState; Index index; + LifecyclePolicy lifecyclePolicy; @Before public void setupClusterState() { @@ -35,19 +45,25 @@ public void setupClusterState() { .put(LifecycleSettings.LIFECYCLE_NAME, policy)) .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); index = indexMetadata.getIndex(); + lifecyclePolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policy); + IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata( + Collections.singletonMap(policy, new LifecyclePolicyMetadata(lifecyclePolicy, Collections.emptyMap())), + OperationMode.RUNNING); MetaData metaData = MetaData.builder() .persistentSettings(settings(Version.CURRENT).build()) .put(IndexMetaData.builder(indexMetadata)) + .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta) .build(); clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build(); } public void testExecuteSuccessfullyMoved() { - StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name"); - StepKey nextStepKey = new StepKey("next-phase", "next-action", "next-name"); long now = randomNonNegativeLong(); + List steps = lifecyclePolicy.toSteps(null, () -> now); + StepKey currentStepKey = steps.get(0).getKey(); + StepKey nextStepKey = steps.get(0).getNextStepKey(); - setStateToKey(currentStepKey); + setStateToKey(currentStepKey, now); SetOnce changed = new SetOnce<>(); MoveToNextStepUpdateTask.Listener listener = (c) -> changed.set(true); @@ -66,7 +82,7 @@ public void testExecuteDifferentCurrentStep() { StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name"); StepKey notCurrentStepKey = new StepKey("not-current", "not-current", "not-current"); long now = randomNonNegativeLong(); - setStateToKey(notCurrentStepKey); + setStateToKey(notCurrentStepKey, now); MoveToNextStepUpdateTask.Listener listener = (c) -> { }; MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, null, () -> now, listener); @@ -77,7 +93,7 @@ public void testExecuteDifferentCurrentStep() { public void testExecuteDifferentPolicy() { StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name"); long now = randomNonNegativeLong(); - setStateToKey(currentStepKey); + setStateToKey(currentStepKey, now); setStatePolicy("not-" + policy); MoveToNextStepUpdateTask.Listener listener = (c) -> {}; MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, null, () -> now, listener); @@ -86,11 +102,12 @@ public void testExecuteDifferentPolicy() { } public void testExecuteSuccessfulMoveWithInvalidNextStep() { - StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name"); - StepKey invalidNextStep = new StepKey("next-invalid", "next-invalid", "next-invalid"); long now = randomNonNegativeLong(); + List steps = lifecyclePolicy.toSteps(null, () -> now); + StepKey currentStepKey = steps.get(0).getKey(); + StepKey invalidNextStep = new StepKey("next-invalid", "next-invalid", "next-invalid"); - setStateToKey(currentStepKey); + setStateToKey(currentStepKey, now); SetOnce changed = new SetOnce<>(); MoveToNextStepUpdateTask.Listener listener = (c) -> changed.set(true); @@ -108,7 +125,7 @@ public void testExecuteSuccessfulMoveWithInvalidNextStep() { public void testClusterProcessedWithNoChange() { StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name"); long now = randomNonNegativeLong(); - setStateToKey(currentStepKey); + setStateToKey(currentStepKey, now); SetOnce changed = new SetOnce<>(); MoveToNextStepUpdateTask.Listener listener = (c) -> changed.set(true); MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, null, () -> now, listener); @@ -121,7 +138,7 @@ public void testOnFailure() { StepKey nextStepKey = new StepKey("next-phase", "next-action", "next-name"); long now = randomNonNegativeLong(); - setStateToKey(currentStepKey); + setStateToKey(currentStepKey, now); SetOnce changed = new SetOnce<>(); MoveToNextStepUpdateTask.Listener listener = (c) -> changed.set(true); @@ -141,12 +158,16 @@ private void setStatePolicy(String policy) { .put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), index.getName())).build(); } - private void setStateToKey(StepKey stepKey) { + private void setStateToKey(StepKey stepKey, long now) { clusterState = ClusterState.builder(clusterState) .metaData(MetaData.builder(clusterState.metaData()) .updateSettings(Settings.builder() + .put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, "{\"actions\":{\"TEST_ACTION\":{}}}") .put(LifecycleSettings.LIFECYCLE_PHASE, stepKey.getPhase()) + .put(LifecycleSettings.LIFECYCLE_PHASE_TIME, now) .put(LifecycleSettings.LIFECYCLE_ACTION, stepKey.getAction()) - .put(LifecycleSettings.LIFECYCLE_STEP, stepKey.getName()).build(), index.getName())).build(); + .put(LifecycleSettings.LIFECYCLE_ACTION_TIME, now) + .put(LifecycleSettings.LIFECYCLE_STEP, stepKey.getName()) + .put(LifecycleSettings.LIFECYCLE_STEP_TIME, now).build(), index.getName())).build(); } } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java index db768495d5937..bed8d2313653b 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -53,7 +54,7 @@ public void testGetFirstStep() { String policyName = randomAlphaOfLengthBetween(2, 10); Step expectedFirstStep = new MockStep(MOCK_STEP_KEY, null); Map firstStepMap = Collections.singletonMap(policyName, expectedFirstStep); - PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, null); + PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, null, NamedXContentRegistry.EMPTY); Step actualFirstStep = registry.getFirstStep(policyName); assertThat(actualFirstStep, sameInstance(expectedFirstStep)); } @@ -62,7 +63,7 @@ public void testGetFirstStepUnknownPolicy() { String policyName = randomAlphaOfLengthBetween(2, 10); Step expectedFirstStep = new MockStep(MOCK_STEP_KEY, null); Map firstStepMap = Collections.singletonMap(policyName, expectedFirstStep); - PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, null); + PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, null, NamedXContentRegistry.EMPTY); Step actualFirstStep = registry.getFirstStep(policyName + "unknown"); assertNull(actualFirstStep); } @@ -71,7 +72,7 @@ public void testGetStep() { Step expectedStep = new MockStep(MOCK_STEP_KEY, null); Index index = new Index("test", "uuid"); Map> indexSteps = Collections.singletonMap(index, Collections.singletonList(expectedStep)); - PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps); + PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps, NamedXContentRegistry.EMPTY); Step actualStep = registry.getStep(index, MOCK_STEP_KEY); assertThat(actualStep, sameInstance(expectedStep)); } @@ -81,13 +82,13 @@ public void testGetStepErrorStep() { Step expectedStep = new ErrorStep(errorStepKey); Index index = new Index("test", "uuid"); Map> indexSteps = Collections.singletonMap(index, Collections.singletonList(expectedStep)); - PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps); + PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps, NamedXContentRegistry.EMPTY); Step actualStep = registry.getStep(index, errorStepKey); assertThat(actualStep, equalTo(expectedStep)); } public void testGetStepUnknownPolicy() { - PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, Collections.emptyMap()); + PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, Collections.emptyMap(), NamedXContentRegistry.EMPTY); assertNull(registry.getStep(new Index("test", "uuid"), MOCK_STEP_KEY)); } @@ -95,7 +96,7 @@ public void testGetStepUnknownStepKey() { Step expectedStep = new MockStep(MOCK_STEP_KEY, null); Index index = new Index("test", "uuid"); Map> indexSteps = Collections.singletonMap(index, Collections.singletonList(expectedStep)); - PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps); + PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps, NamedXContentRegistry.EMPTY); Step.StepKey unknownStepKey = new Step.StepKey(MOCK_STEP_KEY.getPhase(), MOCK_STEP_KEY.getAction(),MOCK_STEP_KEY.getName() + "not"); assertNull(registry.getStep(index, unknownStepKey)); @@ -145,7 +146,7 @@ public void testUpdateFromNothingToSomethingToNothing() throws Exception { .build(); // start with empty registry - PolicyStepsRegistry registry = new PolicyStepsRegistry(); + PolicyStepsRegistry registry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY); // add new policy registry.update(currentState, client, () -> 0L); @@ -191,7 +192,7 @@ public void testUpdateFromNothingToSomethingToNothing() throws Exception { assertTrue(registry.getStepMap().isEmpty()); } - public void testUpdateChangedPolicy() { + public void testUpdateChangedPolicy() throws Exception { Client client = Mockito.mock(Client.class); Mockito.when(client.settings()).thenReturn(Settings.EMPTY); String policyName = randomAlphaOfLengthBetween(5, 10); @@ -216,7 +217,7 @@ public void testUpdateChangedPolicy() { .metaData(metaData) .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build()) .build(); - PolicyStepsRegistry registry = new PolicyStepsRegistry(); + PolicyStepsRegistry registry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY); // add new policy registry.update(currentState, client, () -> 0L); @@ -285,7 +286,7 @@ public void testUpdatePolicyButNoPhaseChangeIndexStepsDontChange() throws Except .build(); // start with empty registry - PolicyStepsRegistry registry = new PolicyStepsRegistry(); + PolicyStepsRegistry registry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY); // add new policy registry.update(currentState, client, () -> 0L); From 6b3285afaa879755ac265f11087ff80ce06d7c90 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 4 Sep 2018 16:07:15 -0700 Subject: [PATCH 2/2] fix security client --- .../xpack/indexlifecycle/PolicyStepsRegistry.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java index 6446896740d40..a34abe167651f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java @@ -190,7 +190,9 @@ public LifecyclePolicyMetadata read(StreamInput in, String key) { } policyToExecute = new LifecyclePolicy(currentPolicy.getType(), currentPolicy.getName(), phaseMap); } - final List steps = policyToExecute.toSteps(client, nowSupplier); + LifecyclePolicySecurityClient policyClient = new LifecyclePolicySecurityClient(client, + ClientHelper.INDEX_LIFECYCLE_ORIGIN, lifecyclePolicyMap.get(policy).getHeaders()); + final List steps = policyToExecute.toSteps(policyClient, nowSupplier); // Build a list of steps that correspond with the phase the index is currently in final List phaseSteps; if (steps == null) {