From 22c76ccbeaa652b15c300d6b6263887b026fb848 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 3 Apr 2019 16:05:39 -0600 Subject: [PATCH 1/2] Allow ILM to stop if indices have nonexistent policies Prior to the PR, there is a bug in ILM which does not allow ILM to stop if one or more indices have an `index.lifecycle.name` which does not exist - the `operation_mode` will be stuck as `STOPPING` until either the policy is created or the nonexistent policy is removed from those indices. This change allows ILM to stop in this case and makes the logging more clear as to why ILM is not stopping. --- .../TimeSeriesLifecycleActionsIT.java | 22 +++++++ .../indexlifecycle/IndexLifecycleService.java | 63 ++++++++++++------- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 8f07b532769c3..69aca756554ac 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -812,6 +812,28 @@ public void testMoveToInjectedStep() throws Exception { }); } + public void testCanStopILMWithPolicyUsingNonexistentPolicy() throws Exception { + createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(5,15))); + + Request stopILMRequest = new Request("POST", "_ilm/stop"); + assertOK(client().performRequest(stopILMRequest)); + + Request statusRequest = new Request("GET", "_ilm/status"); + assertBusy(() -> { + Response statusResponse = client().performRequest(statusRequest); + assertOK(statusResponse); + Map statusResponseMap = entityAsMap(statusResponse); + String status = (String) statusResponseMap.get("operation_mode"); + assertEquals("STOPPED", status); + }); + + // Re-start ILM so that subsequent tests don't fail + Request startILMReqest = new Request("POST", "_ilm/start"); + assertOK(client().performRequest(startILMReqest)); + } + private void createFullPolicy(TimeValue hotTime) throws IOException { Map hotActions = new HashMap<>(); hotActions.put(SetPriorityAction.NAME, new SetPriorityAction(100)); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java index 34cdbb46a5c3a..adb1bf498f323 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java @@ -43,7 +43,7 @@ * A service which runs the {@link LifecyclePolicy}s associated with indexes. */ public class IndexLifecycleService - implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener { + implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener { private static final Logger logger = LogManager.getLogger(IndexLifecycleService.class); private static final Set IGNORE_ACTIONS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkAction.NAME); private volatile boolean isMaster = false; @@ -111,18 +111,25 @@ public void onMaster() { IndexMetaData idxMeta = cursor.value; String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()); if (Strings.isNullOrEmpty(policyName) == false) { - StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(LifecycleExecutionState.fromIndexMetadata(idxMeta)); - if (OperationMode.STOPPING == currentMode && - stepKey != null && - IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction()) == false) { - logger.info("skipping policy [{}] for index [{}]. stopping Index Lifecycle execution", - policyName, idxMeta.getIndex().getName()); - continue; + final LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMeta); + StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState); + + if (OperationMode.STOPPING == currentMode) { + if (stepKey != null && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction())) { + logger.info("waiting to stop ILM because index [{}] with policy [{}] is currently in action [{}]", + idxMeta.getIndex().getName(), policyName, stepKey.getAction()); + lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey); + safeToStop = false; + } else { + logger.info("skipping policy execution for index [{}] with policy [{}] because ILM is stopping", + idxMeta.getIndex().getName(), policyName); + } + } else { + lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey); } - lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey); - safeToStop = false; // proven false! } } + if (safeToStop && OperationMode.STOPPING == currentMode) { submitOperationModeUpdate(OperationMode.STOPPED); } @@ -184,7 +191,7 @@ public void clusterChanged(ClusterChangedEvent event) { @Override public void applyClusterState(ClusterChangedEvent event) { if (event.localNodeMaster()) { // only act if we are master, otherwise - // keep idle until elected + // keep idle until elected if (event.state().metaData().custom(IndexLifecycleMetadata.TYPE) != null) { policyRegistry.update(event.state()); } @@ -237,21 +244,33 @@ void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange) IndexMetaData idxMeta = cursor.value; String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()); if (Strings.isNullOrEmpty(policyName) == false) { - StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(LifecycleExecutionState.fromIndexMetadata(idxMeta)); - if (OperationMode.STOPPING == currentMode && stepKey != null - && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction()) == false) { - logger.info("skipping policy [" + policyName + "] for index [" + idxMeta.getIndex().getName() - + "]. stopping Index Lifecycle execution"); - continue; - } - if (fromClusterStateChange) { - lifecycleRunner.runPolicyAfterStateChange(policyName, idxMeta); + final LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMeta); + StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState); + + if (OperationMode.STOPPING == currentMode) { + if (stepKey != null && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction())) { + logger.info("waiting to stop ILM because index [{}] with policy [{}] is currently in action [{}]", + idxMeta.getIndex().getName(), policyName, stepKey.getAction()); + if (fromClusterStateChange) { + lifecycleRunner.runPolicyAfterStateChange(policyName, idxMeta); + } else { + lifecycleRunner.runPeriodicStep(policyName, idxMeta); + } + safeToStop = false; + } else { + logger.info("skipping policy execution for index [{}] with policy [{}] because ILM is stopping", + idxMeta.getIndex().getName(), policyName); + } } else { - lifecycleRunner.runPeriodicStep(policyName, idxMeta); + if (fromClusterStateChange) { + lifecycleRunner.runPolicyAfterStateChange(policyName, idxMeta); + } else { + lifecycleRunner.runPeriodicStep(policyName, idxMeta); + } } - safeToStop = false; // proven false! } } + if (safeToStop && OperationMode.STOPPING == currentMode) { submitOperationModeUpdate(OperationMode.STOPPED); } From 7264690d80ded090da692b8314e60c9106a76c1e Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 3 Apr 2019 16:44:20 -0600 Subject: [PATCH 2/2] Add comments per review --- .../xpack/indexlifecycle/IndexLifecycleService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java index adb1bf498f323..2d09d0184f489 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java @@ -119,6 +119,7 @@ public void onMaster() { logger.info("waiting to stop ILM because index [{}] with policy [{}] is currently in action [{}]", idxMeta.getIndex().getName(), policyName, stepKey.getAction()); lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey); + // ILM is trying to stop, but this index is in a Shrink action (or other dangerous action) so we can't stop safeToStop = false; } else { logger.info("skipping policy execution for index [{}] with policy [{}] because ILM is stopping", @@ -256,6 +257,7 @@ void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange) } else { lifecycleRunner.runPeriodicStep(policyName, idxMeta); } + // ILM is trying to stop, but this index is in a Shrink action (or other dangerous action) so we can't stop safeToStop = false; } else { logger.info("skipping policy execution for index [{}] with policy [{}] because ILM is stopping",