From 542460ae91f216a32349a2ad1444e35843039839 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 23 Aug 2018 15:41:43 -0700 Subject: [PATCH 1/4] add full policy integration test - this adds an integration test that runs through a policy with all the actions defined. - adds a test specific to a policy having just a rollover action - bumps the node count to 4 --- x-pack/plugin/ilm/build.gradle | 2 +- .../TimeSeriesLifecycleActionsIT.java | 89 ++++++++++++++++--- 2 files changed, 76 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle index 6c3880d78d893..2cda35c24a9f2 100644 --- a/x-pack/plugin/ilm/build.gradle +++ b/x-pack/plugin/ilm/build.gradle @@ -21,7 +21,7 @@ dependencies { check.dependsOn 'qa:with-security:integTestRunner' integTestCluster { - numNodes = 2 + numNodes = 4 clusterName = 'ilm' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 30737168c1fbe..2a8acf3934060 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -9,12 +9,12 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction; import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction; @@ -24,6 +24,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings; import org.elasticsearch.xpack.core.indexlifecycle.Phase; import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction; +import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction; import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep; @@ -31,6 +32,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -46,20 +48,9 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase { private String policy; @Before - public void refreshIndex() throws IOException { + public void refreshIndex() { index = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); policy = randomAlphaOfLength(5); - Request request = new Request("PUT", "/_cluster/settings"); - XContentBuilder pollIntervalEntity = JsonXContent.contentBuilder(); - pollIntervalEntity.startObject(); - { - pollIntervalEntity.startObject("transient"); - { - pollIntervalEntity.field(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s"); - }pollIntervalEntity.endObject(); - } pollIntervalEntity.endObject(); - request.setJsonEntity(Strings.toString(pollIntervalEntity)); - assertOK(adminClient().performRequest(request)); } public static void updatePolicy(String indexName, String policy) throws IOException { @@ -67,6 +58,62 @@ public static void updatePolicy(String indexName, String policy) throws IOExcept client().performRequest(request); } + public void testFullPolicy() throws Exception { + String originalIndex = index + "-000001"; + String shrunkenOriginalIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + originalIndex; + String secondIndex = index + "-000002"; + createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.routing.allocation.include._name", "node-0") + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias")); + + // create policy + Map warmActions = new HashMap<>(); + warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1)); + warmActions.put(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "node-1,node-2"), null, null)); + warmActions.put(ShrinkAction.NAME, new ShrinkAction(1)); + Map phases = new HashMap<>(); + phases.put("hot", new Phase("hot", TimeValue.ZERO, singletonMap(RolloverAction.NAME, + new RolloverAction(null, null, 1L)))); + phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions)); + phases.put("cold", new Phase("cold", TimeValue.ZERO, singletonMap(AllocateAction.NAME, + new AllocateAction(1, singletonMap("_name", "node-3"), null, null)))); + phases.put("delete", new Phase("delete", TimeValue.ZERO, singletonMap(DeleteAction.NAME, new DeleteAction()))); + LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); + // PUT policy + XContentBuilder builder = jsonBuilder(); + lifecyclePolicy.toXContent(builder, null); + final StringEntity entity = new StringEntity( + "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON); + Request request = new Request("PUT", "_ilm/" + policy); + request.setEntity(entity); + client().performRequest(request); + // update policy on index + updatePolicy(originalIndex, policy); + // index document {"foo": "bar"} to trigger rollover + index(client(), originalIndex, "_id", "foo", "bar"); + assertBusy(() -> assertTrue(indexExists(secondIndex))); + assertBusy(() -> assertFalse(indexExists(shrunkenOriginalIndex))); + assertBusy(() -> assertFalse(indexExists(originalIndex))); + } + + public void testRolloverAction() throws Exception { + String originalIndex = index + "-000001"; + String secondIndex = index + "-000002"; + createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias")); + + // create policy + createNewSingletonPolicy("hot", new RolloverAction(null, null, 1L)); + // update policy on index + updatePolicy(originalIndex, policy); + // index document {"foo": "bar"} to trigger rollover + index(client(), originalIndex, "_id", "foo", "bar"); + assertBusy(() -> assertTrue(indexExists(secondIndex))); + assertBusy(() -> assertTrue(indexExists(originalIndex))); + } + public void testAllocateOnlyAllocation() throws Exception { createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); @@ -183,12 +230,26 @@ private void createNewSingletonPolicy(String phaseName, LifecycleAction action) private void createIndexWithSettings(String index, Settings.Builder settings) throws IOException { // create the test-index index - createIndex(index, settings.build()); + Request request = new Request("PUT", "/" + index); + request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings.build()) + + ", \"aliases\" : { \"alias\": { \"is_write_index\": true } } }"); + client().performRequest(request); // wait for the shards to initialize ensureGreen(index); } + private static void index(RestClient client, String index, String id, Object... fields) throws IOException { + XContentBuilder document = jsonBuilder().startObject(); + for (int i = 0; i < fields.length; i += 2) { + document.field((String) fields[i], fields[i + 1]); + } + document.endObject(); + final Request request = new Request("POST", "/" + index + "/_doc/" + id); + request.setJsonEntity(Strings.toString(document)); + assertOK(client.performRequest(request)); + } + @SuppressWarnings("unchecked") private Map getOnlyIndexSettings(String index) throws IOException { Map response = (Map) getIndexSettings(index).get(index); From 5880a438ff3fc000c7a5a3d3a00bdf622e5f1a3c Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 6 Sep 2018 13:38:53 -0700 Subject: [PATCH 2/4] change replica count in final allocation --- .../xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 2a8acf3934060..c4e25153628eb 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -77,7 +77,7 @@ public void testFullPolicy() throws Exception { new RolloverAction(null, null, 1L)))); phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions)); phases.put("cold", new Phase("cold", TimeValue.ZERO, singletonMap(AllocateAction.NAME, - new AllocateAction(1, singletonMap("_name", "node-3"), null, null)))); + new AllocateAction(0, singletonMap("_name", "node-3"), null, null)))); phases.put("delete", new Phase("delete", TimeValue.ZERO, singletonMap(DeleteAction.NAME, new DeleteAction()))); LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); // PUT policy From 2afafe5260314135f0ea171767c9e75bbd6e88c9 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 17 Sep 2018 18:18:39 -0700 Subject: [PATCH 3/4] increase poll --- x-pack/plugin/ilm/build.gradle | 2 +- .../xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle index 2cda35c24a9f2..6085d02a2b9fb 100644 --- a/x-pack/plugin/ilm/build.gradle +++ b/x-pack/plugin/ilm/build.gradle @@ -28,7 +28,7 @@ integTestCluster { setting 'xpack.monitoring.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.ilm.enabled', 'true' - setting 'indices.lifecycle.poll_interval', '500ms' + setting 'indices.lifecycle.poll_interval', '1500ms' module project(xpackModule('core')) } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index c4e25153628eb..4fb6ce9d91dec 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -55,7 +55,7 @@ public void refreshIndex() { public static void updatePolicy(String indexName, String policy) throws IOException { Request request = new Request("PUT", "/" + indexName + "/_ilm/" + policy); - client().performRequest(request); + assertOK(client().performRequest(request)); } public void testFullPolicy() throws Exception { @@ -87,7 +87,7 @@ public void testFullPolicy() throws Exception { "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON); Request request = new Request("PUT", "_ilm/" + policy); request.setEntity(entity); - client().performRequest(request); + assertOK(client().performRequest(request)); // update policy on index updatePolicy(originalIndex, policy); // index document {"foo": "bar"} to trigger rollover From 8a31c482a3609701c2f6f19bc82df4fd63209e1a Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 18 Sep 2018 08:17:27 -0700 Subject: [PATCH 4/4] upgdate build.gradle with more poll --- x-pack/plugin/ilm/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle index 6085d02a2b9fb..66836a45c97d2 100644 --- a/x-pack/plugin/ilm/build.gradle +++ b/x-pack/plugin/ilm/build.gradle @@ -28,7 +28,7 @@ integTestCluster { setting 'xpack.monitoring.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.ilm.enabled', 'true' - setting 'indices.lifecycle.poll_interval', '1500ms' + setting 'indices.lifecycle.poll_interval', '2500ms' module project(xpackModule('core')) }