[7.x] Add simulate template composition API _index_template/_simulate_index/{name} (#55686) (#55922)

This adds a new api to simulate matching the given index name against the
 index templates in the system.

The syntax for the new API takes the following form:

POST _index_template/_simulate_index/{index_name}
{
  "index_patterns": ["logs-*"],
  "priority": 15,
  "template": {
	"settings": {
		"number_of_shards": 3
	}
       ...
   }
}

Where the body is optional, but we support the entire body used by the
PUT _index_template/{name} api. When the body is specified we'll simulate
matching the given index against a system that'd have the given index
template together with the index templates that exist in the system.

The response, in both cases, will return the matching template's resolved
settings, mappings and aliases, together with a special field that'll print any
overlapping templates and their corresponding index patterns.

(cherry picked from commit 1a5845e)
Signed-off-by: Andrei Dan <[email protected]>

Author: andreidan

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -861,6 +861,7 @@ public void testApiNamingConventions() throws Exception {
             "indices.create_data_stream",
             "indices.get_data_streams",
             "indices.delete_data_stream",
+            "indices.simulate_index_template"
         };
         //These API are not required for high-level client feature completeness
         String[] notRequiredApi = new String[] {

rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json

@@ -0,0 +1,35 @@
+{
+  "indices.simulate_index_template":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "description": "Simulate matching the given index name against the index templates in the system"
+    },
+    "stability":"stable",
+    "url":{
+      "paths":[
+        {
+          "path":"/_index_template/_simulate_index/{name}",
+          "methods":[
+            "POST"
+          ],
+          "parts":{
+            "name":{
+              "type":"string",
+              "description":"The name of the index (it must be a concrete index name)"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
+    },
+    "body":{
+      "description":"New index template definition, which will be included in the simulation, as if it already exists in the system",
+      "required":false
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml

@@ -0,0 +1,181 @@
+---
+"Simulate index template without new template in the body":
+  - skip:
+      version: " - 7.99.99"
+      reason: "simulate index template API has not been backported"
+      features: allowed_warnings
+
+  - do:
+      allowed_warnings:
+        - "index template [test] has index patterns [te*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
+        name: test
+        body:
+          index_patterns: te*
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                field:
+                  type: keyword
+
+  - do:
+      indices.simulate_index_template:
+        name: test
+
+  - match: {template.settings.index.number_of_shards: "1"}
+  - match: {template.settings.index.number_of_replicas: "0"}
+  - match: {template.mappings._doc.properties.field.type: "keyword"}
+  - match: {overlapping: []}
+
+---
+"Simulate index template specifying a new template":
+  - skip:
+      version: " - 7.99.99"
+      reason: "simulate index template API has not been backported"
+      features: allowed_warnings
+
+  - do:
+      allowed_warnings:
+        - "index template [test] has index patterns [te*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
+        name: existing_test
+        body:
+          index_patterns: te*
+          priority: 10
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                field:
+                  type: keyword
+
+  - do:
+      cluster.put_component_template:
+        name: ct
+        body:
+          template:
+            settings:
+              index.number_of_replicas: 2
+            mappings:
+              properties:
+                ct_field:
+                  type: keyword
+
+  - do:
+      indices.simulate_index_template:
+        name: test
+        body:
+          index_patterns: te*
+          priority: 15
+          template:
+            settings:
+              index.blocks.write: true
+            aliases:
+              test_alias: {}
+          composed_of: ["ct"]
+
+  - match: {template.settings.index.blocks.write: "true"}
+  - match: {template.settings.index.number_of_replicas: "2"}
+  - match: {template.mappings._doc.properties.ct_field.type: "keyword"}
+  - match: {overlapping.0.name: existing_test}
+  - match: {overlapping.0.index_patterns: ["te*"]}
+  - length: {template.aliases: 1}
+  - is_true: template.aliases.test_alias
+
+---
+"Simulate index template with index not matching any template":
+  - skip:
+      version: " - 7.99.99"
+      reason: "simulate index template API has not been backported"
+      features: allowed_warnings
+
+  - do:
+      allowed_warnings:
+        - "index template [test] has index patterns [te*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
+        name: test
+        body:
+          index_patterns: te*
+          priority: 10
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                field:
+                  type: keyword
+
+  - do:
+      indices.simulate_index_template:
+        name: will_not_match
+
+  - match: {body: null}
+
+---
+"Simulate index matches overlapping V1 and V2 templates":
+  - skip:
+      version: " - 7.99.99"
+      reason: "simulate index template API has not been backported"
+      features: allowed_warnings
+
+  - do:
+      indices.put_template:
+        name: v1_template
+        body:
+          index_patterns: [t*, t1*]
+          settings:
+            number_of_shards:   5
+
+  - do:
+      allowed_warnings:
+        - "index template [v2_template] has index patterns [te*] matching patterns from existing older templates [v1_template] with patterns
+        (v1_template => [t*, t1*]); this template [v2_template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: v2_template
+        body:
+          index_patterns: te*
+          priority: 10
+          template:
+            settings:
+              number_of_shards:   10
+              number_of_replicas: 2
+            mappings:
+              properties:
+                field:
+                  type: text
+
+  - do:
+      allowed_warnings:
+        - "index template [winning_v2_template] has index patterns [te*] matching patterns from existing older templates [v1_template] with patterns
+        (v1_template => [t*, t1*]); this template [winning_v2_template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: winning_v2_template
+        body:
+          index_patterns: te*
+          priority: 20
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                field:
+                  type: keyword
+
+  - do:
+      indices.simulate_index_template:
+        name: test
+
+  - match: {template.settings.index.number_of_shards: "1"}
+  - match: {template.settings.index.number_of_replicas: "0"}
+  - match: {template.mappings._doc.properties.field.type: "keyword"}
+  - match: {overlapping.0.name: v1_template}
+  - match: {overlapping.0.index_patterns: ["t*", "t1*"]}
+  - match: {overlapping.1.name: v2_template}
+  - match: {overlapping.1.index_patterns: ["te*"]}

server/src/main/java/org/elasticsearch/action/ActionModule.java

@@ -28,9 +28,6 @@
 import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.cluster.configuration.TransportClearVotingConfigExclusionsAction;
-import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
-import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
-import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction;
@@ -109,6 +106,9 @@
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
+import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
+import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
@@ -163,6 +163,8 @@
 import org.elasticsearch.action.admin.indices.template.get.TransportGetComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.get.TransportGetIndexTemplateV2Action;
 import org.elasticsearch.action.admin.indices.template.get.TransportGetIndexTemplatesAction;
+import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.template.post.TransportSimulateIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action;
@@ -261,11 +263,9 @@
 import org.elasticsearch.rest.action.admin.cluster.RestClusterStatsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestClusterUpdateSettingsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestCreateSnapshotAction;
-import org.elasticsearch.rest.action.admin.indices.RestDeleteDataStreamAction;
 import org.elasticsearch.rest.action.admin.cluster.RestDeleteRepositoryAction;
 import org.elasticsearch.rest.action.admin.cluster.RestDeleteSnapshotAction;
 import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
-import org.elasticsearch.rest.action.admin.indices.RestGetDataStreamsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction;
@@ -278,7 +278,6 @@
 import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
-import org.elasticsearch.rest.action.admin.indices.RestCreateDataStreamAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.RestReloadSecureSettingsAction;
@@ -289,15 +288,18 @@
 import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction;
 import org.elasticsearch.rest.action.admin.indices.RestClearIndicesCacheAction;
 import org.elasticsearch.rest.action.admin.indices.RestCloseIndexAction;
+import org.elasticsearch.rest.action.admin.indices.RestCreateDataStreamAction;
 import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction;
 import org.elasticsearch.rest.action.admin.indices.RestDeleteComponentTemplateAction;
+import org.elasticsearch.rest.action.admin.indices.RestDeleteDataStreamAction;
 import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexAction;
 import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexTemplateV2Action;
 import org.elasticsearch.rest.action.admin.indices.RestFlushAction;
 import org.elasticsearch.rest.action.admin.indices.RestForceMergeAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetAliasesAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetComponentTemplateAction;
+import org.elasticsearch.rest.action.admin.indices.RestGetDataStreamsAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateV2Action;
@@ -311,6 +313,7 @@
 import org.elasticsearch.rest.action.admin.indices.RestIndicesShardStoresAction;
 import org.elasticsearch.rest.action.admin.indices.RestIndicesStatsAction;
 import org.elasticsearch.rest.action.admin.indices.RestOpenIndexAction;
+import org.elasticsearch.rest.action.admin.indices.RestSimulateIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateV2Action;
@@ -561,6 +564,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
             actions.register(PutIndexTemplateV2Action.INSTANCE, TransportPutIndexTemplateV2Action.class);
             actions.register(GetIndexTemplateV2Action.INSTANCE, TransportGetIndexTemplateV2Action.class);
             actions.register(DeleteIndexTemplateV2Action.INSTANCE, TransportDeleteIndexTemplateV2Action.class);
+            actions.register(SimulateIndexTemplateAction.INSTANCE, TransportSimulateIndexTemplateAction.class);
         }
         actions.register(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class);
         actions.register(RefreshAction.INSTANCE, TransportRefreshAction.class);
@@ -703,6 +707,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
             registerHandler.accept(new RestPutIndexTemplateV2Action());
             registerHandler.accept(new RestGetIndexTemplateV2Action());
             registerHandler.accept(new RestDeleteIndexTemplateV2Action());
+            registerHandler.accept(new RestSimulateIndexTemplateAction());
         }
 
         registerHandler.accept(new RestPutMappingAction());

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.template.post;
+
+import org.elasticsearch.action.ActionType;
+
+public class SimulateIndexTemplateAction extends ActionType<SimulateIndexTemplateResponse> {
+
+    public static final SimulateIndexTemplateAction INSTANCE = new SimulateIndexTemplateAction();
+    public static final String NAME = "indices:admin/index_template/simulate_index";
+
+    private SimulateIndexTemplateAction() {
+        super(NAME, SimulateIndexTemplateResponse::new);
+    }
+
+}