Prohibit the usage of create index api in namespaces managed by data stream templates (#62574)

Backport of #62527 to 7.x branch.

This commit adds validation that prohibits the creation of regular indices
in the namespace of templates with data streams enabled.

It shouldn't be possible to create ordinary indices when the name of the index
matches with a composable index template that enables data streams. Auto creation
has logic that creates data streams instead of regular indices. However validation
logic for the create index api was missing.

Author: martijnvg

server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java

@@ -48,7 +48,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
- * Api that auto creates an index that originate from requests that write into an index that doesn't yet exist.
+ * Api that auto creates an index or data stream that originate from requests that write into an index that doesn't yet exist.
  */
 public final class AutoCreateAction extends ActionType<CreateIndexResponse> {
 

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -508,6 +508,13 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu
                                                                                     throws Exception {
         logger.debug("applying create index request using composable template [{}]", templateName);
 
+        ComposableIndexTemplate template = currentState.getMetadata().templatesV2().get(templateName);
+        if (request.dataStreamName() == null && template.getDataStreamTemplate() != null) {
+           throw new IllegalArgumentException("cannot create index with name [" + request.index() +
+               "], because it matches with template [" + templateName + "] that creates data streams only, " +
+               "use create data stream api instead");
+        }
+
         final List<Map<String, Map<String, Object>>> mappings =
             collectV2Mappings(request.mappings(), currentState, templateName, xContentRegistry, request.index());
         final Settings aggregatedIndexSettings =

x-pack/plugin/ccr/qa/security/follower-roles.yml

@@ -2,7 +2,7 @@ ccruser:
   cluster:
     - manage_ccr
   indices:
-    - names: [ 'allowed-index', 'forget-follower', 'logs-eu-*' ]
+    - names: [ 'allowed-index', 'forget-follower', 'logs-eu*' ]
       privileges:
         - monitor
         - read

x-pack/plugin/ccr/qa/security/leader-roles.yml

@@ -2,7 +2,7 @@ ccruser:
   cluster:
     - read_ccr
   indices:
-    - names: [ 'allowed-index', 'clean-leader', 'forget-leader', 'logs-eu-*' ]
+    - names: [ 'allowed-index', 'clean-leader', 'forget-leader', 'logs-eu*' ]
       privileges:
         - monitor
         - read

x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java

@@ -132,8 +132,8 @@ public void testFollowIndex() throws Exception {
 
     public void testAutoFollowPatterns() throws Exception {
         assumeFalse("Test should only run when both clusters are running", "leader".equals(targetCluster));
-        String allowedIndex = "logs-eu-20190101";
-        String disallowedIndex = "logs-us-20190101";
+        String allowedIndex = "logs-eu_20190101";
+        String disallowedIndex = "logs-us_20190101";
 
         {
             Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern");
@@ -143,7 +143,7 @@ public void testAutoFollowPatterns() throws Exception {
         }
 
         Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern");
-        request.setJsonEntity("{\"leader_index_patterns\": [\"logs-eu-*\"], \"remote_cluster\": \"leader_cluster\"}");
+        request.setJsonEntity("{\"leader_index_patterns\": [\"logs-eu*\"], \"remote_cluster\": \"leader_cluster\"}");
         assertOK(client().performRequest(request));
 
         try (RestClient leaderClient = buildLeaderClient()) {

x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_basic.yml

@@ -399,3 +399,31 @@ setup:
       indices.delete_data_stream:
         name: logs-foobar
   - is_true: acknowledged
+
+---
+"Create index into a namespace that is governed by a data stream template":
+  - skip:
+      version: " - 7.99.99"
+      reason: "adjust until #62527 is fully backported"
+      features: allowed_warnings
+
+  - do:
+      allowed_warnings:
+        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: generic_logs_template
+        body:
+          index_patterns: logs-*
+          data_stream: {}
+
+  # test response status code
+  - do:
+      catch: bad_request
+      indices.create:
+        index: logs-foobar
+
+  # test error message
+  - do:
+      catch: /cannot create index with name \[logs-foobar\], because it matches with template \[generic_logs_template\] that creates data streams only, use create data stream api instead/
+      indices.create:
+        index: logs-foobar

x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml

@@ -15,7 +15,7 @@ setup:
         body:  >
           {
             "indices": [
-              { "names": ["simple*"], "privileges": ["read", "write", "create_index", "view_index_metadata", "monitor", "delete_index"] }
+              { "names": ["simple*", "easy*"], "privileges": ["read", "write", "create_index", "view_index_metadata", "monitor", "delete_index"] }
             ]
           }
 
@@ -66,7 +66,7 @@ setup:
       indices.put_index_template:
         name: my-template1
         body:
-          index_patterns: [s*, create-doc-data-stream1, write-data-stream1]
+          index_patterns: [s*, easy-data-stream1, create-doc-data-stream1, write-data-stream1]
           template:
             mappings:
               properties:
@@ -174,27 +174,27 @@ teardown:
 
   - do: # superuser
       indices.create_data_stream:
-        name: simple-data-stream1
+        name: easy-data-stream1
   - is_true: acknowledged
 
   - do: # superuser
       indices.create:
-        index: simple-index
+        index: easy-index
         body:
           aliases:
-            simple-alias: {}
+            easy-alias: {}
 
   - do:
       headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
       indices.get_alias:
-        name: simple*
+        name: easy*
 
-  - match: {simple-index.aliases.simple-alias: {}}
-  - is_false: simple-data-stream1
+  - match: {easy-index.aliases.easy-alias: {}}
+  - is_false: easy-data-stream1
 
   - do: # superuser
       indices.delete_data_stream:
-        name: simple-data-stream1
+        name: easy-data-stream1
   - is_true: acknowledged
 
 ---