Add ILM history store index

client/rest-high-level/build.gradle

@@ -138,6 +138,7 @@ testClusters.all {
   setting 'xpack.security.authc.realms.pki.pki1.delegation.enabled', 'true'
 
   setting 'indices.lifecycle.poll_interval', '1000ms'
+  setting 'index.lifecycle.history_index_enabled', 'false'
   keystore 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
   extraConfigFile 'roles.yml', file('roles.yml')
   user username: System.getProperty('tests.rest.cluster.username', 'test_user'),

docs/build.gradle

@@ -60,6 +60,7 @@ testClusters.integTest {
   extraConfigFile 'hunspell/en_US/en_US.dic', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic')
   // Whitelist reindexing from the local node so we can test it.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
+  setting 'index.lifecycle.history_index_enabled', 'false'
 
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'

docs/plugins/analysis-icu.asciidoc

@@ -368,7 +368,7 @@ PUT my_index
   }
 }
 
-GET _search <3>
+GET /my_index/_search <3>
 {
   "query": {
     "match": {

docs/reference/indices/rollover-index.asciidoc

@@ -428,7 +428,7 @@ PUT logs/_doc/2 <2>
 //////////////////////////
 [source,console]
 --------------------------------------------------
-GET _alias
+GET my_logs_index-000001,my_logs_index-000002/_alias
 --------------------------------------------------
 // TEST[continued]
 //////////////////////////

docs/reference/settings/ilm-settings.asciidoc

@@ -14,6 +14,11 @@ ILM REST API endpoints and functionality. Defaults to `true`.
 (<<time-units, time units>>) How often {ilm} checks for indices that meet policy
 criteria. Defaults to `10m`.
 
+`index.lifecycle.history_index_enabled`::
+Whether ILM's history index is enabled. If enabled, ILM will record the
+history of actions taken as part of ILM policies to the `ilm-history-*`
+indices. Defaults to `true`.
+
 ==== Index level settings
 These index-level {ilm-init} settings are typically configured through index
 templates. For more information, see <<ilm-gs-create-policy>>.

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -75,6 +75,7 @@
 import java.security.cert.CertificateException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -448,6 +449,13 @@ protected boolean preserveILMPoliciesUponCompletion() {
         return false;
     }
 
+    /**
+     * A set of ILM policies that should be preserved between runs.
+     */
+    protected Set<String> preserveILMPolicyIds() {
+        return Collections.singleton("ilm-history-ilm-policy");
+    }
+
     /**
      * Returns whether to preserve auto-follow patterns. Defaults to not
      * preserving them. Only runs at all if xpack is installed on the cluster
@@ -545,7 +553,7 @@ private void wipeCluster() throws Exception {
         }
 
         if (hasXPack && false == preserveILMPoliciesUponCompletion()) {
-            deleteAllILMPolicies();
+            deleteAllILMPolicies(preserveILMPolicyIds());
         }
 
         if (hasXPack && false == preserveAutoFollowPatternsUponCompletion()) {
@@ -680,7 +688,7 @@ private void waitForPendingRollupTasks() throws Exception {
         waitForPendingTasks(adminClient(), taskName -> taskName.startsWith("xpack/rollup/job") == false);
     }
 
-    private static void deleteAllILMPolicies() throws IOException {
+    private static void deleteAllILMPolicies(Set<String> exclusions) throws IOException {
         Map<String, Object> policies;
 
         try {
@@ -699,9 +707,15 @@ private static void deleteAllILMPolicies() throws IOException {
             return;
         }
 
-        for (String policyName : policies.keySet()) {
-            adminClient().performRequest(new Request("DELETE", "/_ilm/policy/" + policyName));
-        }
+        policies.keySet().stream()
+            .filter(p -> exclusions.contains(p) == false)
+            .forEach(policyName -> {
+                try {
+                    adminClient().performRequest(new Request("DELETE", "/_ilm/policy/" + policyName));
+                } catch (IOException e) {
+                    throw new RuntimeException("failed to delete policy: " + policyName, e);
+                }
+            });
     }
 
     private static void deleteAllSLMPolicies() throws IOException {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java

@@ -19,6 +19,7 @@ public class LifecycleSettings {
     public static final String LIFECYCLE_INDEXING_COMPLETE = "index.lifecycle.indexing_complete";
     public static final String LIFECYCLE_ORIGINATION_DATE = "index.lifecycle.origination_date";
     public static final String LIFECYCLE_PARSE_ORIGINATION_DATE = "index.lifecycle.parse_origination_date";
+    public static final String LIFECYCLE_HISTORY_INDEX_ENABLED = "index.lifecycle.history_index_enabled";
 
     public static final String SLM_HISTORY_INDEX_ENABLED = "slm.history_index_enabled";
     public static final String SLM_RETENTION_SCHEDULE = "slm.retention_schedule";
@@ -35,6 +36,8 @@ public class LifecycleSettings {
         Setting.longSetting(LIFECYCLE_ORIGINATION_DATE, -1, -1, Setting.Property.Dynamic, Setting.Property.IndexScope);
     public static final Setting<Boolean> LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING = Setting.boolSetting(LIFECYCLE_PARSE_ORIGINATION_DATE,
         false, Setting.Property.Dynamic, Setting.Property.IndexScope);
+    public static final Setting<Boolean> LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(LIFECYCLE_HISTORY_INDEX_ENABLED,
+        true, Setting.Property.NodeScope);
 
 
     public static final Setting<Boolean> SLM_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(SLM_HISTORY_INDEX_ENABLED, true,

x-pack/plugin/core/src/main/resources/ilm-history-ilm-policy.json

@@ -0,0 +1,18 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_size": "50GB",
+          "max_age": "30d"
+        }
+      }
+    },
+    "delete": {
+      "min_age": "90d",
+      "actions": {
+        "delete": {}
+      }
+    }
+  }
+}

x-pack/plugin/core/src/main/resources/ilm-history.json

@@ -0,0 +1,83 @@
+{
+  "index_patterns": [
+    "ilm-history-${xpack.ilm_history.template.version}*"
+  ],
+  "order": 2147483647,
+  "settings": {
+    "index.number_of_shards": 1,
+    "index.number_of_replicas": 0,
+    "index.auto_expand_replicas": "0-1",
+    "index.lifecycle.name": "ilm-history-ilm-policy",
+    "index.lifecycle.rollover_alias": "ilm-history-${xpack.ilm_history.template.version}",
+    "index.format": 1
+  },
+  "mappings": {
+    "_doc": {
+      "dynamic": false,
+      "properties": {
+        "@timestamp": {
+          "type": "date",
+          "format": "epoch_millis"
+        },
+        "policy": {
+          "type": "keyword"
+        },
+        "index": {
+          "type": "keyword"
+        },
+        "index_age":{
+          "type": "long"
+        },
+        "success": {
+          "type": "boolean"
+        },
+        "state": {
+          "type": "object",
+          "dynamic": true,
+          "properties": {
+            "phase": {
+              "type": "keyword"
+            },
+            "action": {
+              "type": "keyword"
+            },
+            "step": {
+              "type": "keyword"
+            },
+            "failed_step": {
+              "type": "keyword"
+            },
+            "is_auto-retryable_error": {
+              "type": "keyword"
+            },
+            "creation_date": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "phase_time": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "action_time": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "step_time": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "phase_definition": {
+              "type": "text"
+            },
+            "step_info": {
+              "type": "text"
+            }
+          }
+        },
+        "error_details": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}

x-pack/plugin/ilm/qa/multi-node/build.gradle

@@ -25,4 +25,6 @@ testClusters.integTest {
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'indices.lifecycle.poll_interval', '1000ms'
+  setting 'logger.org.elasticsearch.xpack.core.ilm', 'TRACE'
+  setting 'logger.org.elasticsearch.xpack.ilm', 'TRACE'
 }