diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 0d2e7df9c88ee..a9b07ee5d20cf 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -2040,7 +2040,7 @@ public void testDataStreams() throws Exception { CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"@timestamp\":{\"type\":\"date\"}}}"); Template template = new Template(null, mappings, null); ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(Collections.singletonList(dataStreamName), template, - Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate()); + Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate(), null); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name("ds-template").create(true).indexTemplate(indexTemplate); AcknowledgedResponse response = execute(putComposableIndexTemplateRequest, @@ -2119,7 +2119,7 @@ public void testIndexTemplates() throws Exception { Template template = new Template(settings, mappings, Collections.singletonMap("alias", alias)); List pattern = Collections.singletonList("pattern"); ComposableIndexTemplate indexTemplate = - new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null); + new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate); @@ -2166,7 +2166,7 @@ public void testSimulateIndexTemplate() throws Exception { Template template = new Template(settings, mappings, org.elasticsearch.common.collect.Map.of("alias", alias)); List pattern = org.elasticsearch.common.collect.List.of("pattern"); ComposableIndexTemplate indexTemplate = - new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null); + new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate); @@ -2178,7 +2178,7 @@ public void testSimulateIndexTemplate() throws Exception { AliasMetadata simulationAlias = AliasMetadata.builder("simulation-alias").writeIndex(true).build(); ComposableIndexTemplate simulationTemplate = new ComposableIndexTemplate(pattern, new Template(null, null, org.elasticsearch.common.collect.Map.of("simulation-alias", simulationAlias)), Collections.emptyList(), 2L, 1L, - new HashMap<>(), null); + new HashMap<>(), null, null); PutComposableIndexTemplateRequest newIndexTemplateReq = new PutComposableIndexTemplateRequest().name("used-for-simulation").create(true).indexTemplate(indexTemplate); newIndexTemplateReq.indexTemplate(simulationTemplate); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java index 23759e7b09a58..f244f16b6a598 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java @@ -88,6 +88,6 @@ private static ComposableIndexTemplate randomIndexTemplate() { if (randomBoolean()) { dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate(); } - return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate); + return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate, null); } } diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index 1a05b91539c6f..4b00d7ad49dfe 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -4,10 +4,10 @@ Put component template ++++ -Creates or updates a component template. -Component templates are building blocks for constructing <>. -that specify index <>, <>, -and <>. +Creates or updates a component template. +Component templates are building blocks for constructing <>. +that specify index <>, <>, +and <>. [source,console] -------------------------------------------------- @@ -55,10 +55,10 @@ DELETE _component_template/template_* [[put-component-template-api-desc]] ==== {api-description-title} -An index template can be composed of multiple component templates. +An index template can be composed of multiple component templates. To use a component template, specify it in an index template's `composed_of` list. -Component templates are only applied to new data streams and indices -as part of a matching index template. +Component templates are only applied to new data streams and indices +as part of a matching index template. Settings and mappings specified directly in the index template or the <> request override any settings or mappings specified in a component template. @@ -112,6 +112,16 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings] Version number used to manage component templates externally. This number is not automatically generated or incremented by {es}. +`allow_auto_create`:: +(Optional, boolean) +This setting overrides the value of the +<> cluster setting. If set to +`true` in a template, then indices can be automatically created using that +template even if auto-creation of indices is disabled via +`actions.auto_create_index`. If set to `false`, then indices or data streams matching the +template must always be explicitly created, and may never be automatically +created. + `_meta`:: (Optional, object) Optional user metadata about the component template. May have any contents. @@ -157,7 +167,7 @@ To be applied, a component template must be included in an index template's `com [[component-templates-version]] ===== Component template versioning -You can use the `version` parameter to add a version number to a component template. +You can use the `version` parameter to add a version number to a component template. External systems can use these version numbers to simplify template management. The `version` parameter is optional and not automatically generated or used by {es}. @@ -182,7 +192,7 @@ To check the `version`, you can use the <false + * disable the automatic creation on indices. + */ + public void testCannotAutoCreateIndexWhenDisabled() throws IOException { + configureAutoCreateIndex(false); + + // Attempt to add a document to a non-existing index. Auto-creating the index should fail owing to the setting above. + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456"); + indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }"); + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), + containsString("no such index [recipe_kr] and [action.auto_create_index] is [false]") + ); + } + + /** + * Check that automatically creating an index is allowed, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} + * is false, when the index name matches a template and that template has allow_auto_create + * set to true. + */ + public void testCanAutoCreateIndexWhenAllowedByTemplate() throws IOException { + configureAutoCreateIndex(false); + + createTemplateWithAllowAutoCreate(true); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + assertOK(this.indexDocument()); + } + + /** + * Check that automatically creating an index is disallowed when the index name matches a template and that template has + * allow_auto_create explicitly to false, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} + * is set to true. + */ + public void testCannotAutoCreateIndexWhenDisallowedByTemplate() throws IOException { + configureAutoCreateIndex(true); + + createTemplateWithAllowAutoCreate(false); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), + containsString("no such index [composable template [recipe*] forbids index auto creation]") + ); + } + + + private void configureAutoCreateIndex(boolean value) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("transient") + .field(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), value) + .endObject() + .endObject(); + + final Request settingsRequest = new Request("PUT", "_cluster/settings"); + settingsRequest.setJsonEntity(Strings.toString(builder)); + final Response settingsResponse = client().performRequest(settingsRequest); + assertOK(settingsResponse); + } + + private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .array("index_patterns", "recipe*") + .field("allow_auto_create", allowAutoCreate) + .endObject(); + + final Request createTemplateRequest = new Request("PUT", "_index_template/recipe_template"); + createTemplateRequest.setJsonEntity(Strings.toString(builder)); + final Response createTemplateResponse = client().performRequest(createTemplateRequest); + assertOK(createTemplateResponse); + } + + private Response indexDocument() throws IOException { + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456"); + indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }"); + return client().performRequest(indexDocumentRequest); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java index e82be77fc147c..6d2f92eecfe0a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java @@ -25,6 +25,8 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import static org.hamcrest.Matchers.equalTo; + @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class BulkProcessorClusterSettingsIT extends ESIntegTestCase { public void testBulkProcessorAutoCreateRestrictions() throws Exception { @@ -45,7 +47,9 @@ public void testBulkProcessorAutoCreateRestrictions() throws Exception { assertEquals(3, responses.length); assertFalse("Operation on existing index should succeed", responses[0].isFailed()); assertTrue("Missing index should have been flagged", responses[1].isFailed()); - assertEquals("[wontwork] IndexNotFoundException[no such index [wontwork]]", responses[1].getFailureMessage()); + assertThat( + responses[1].getFailureMessage(), + equalTo("[wontwork] IndexNotFoundException[no such index [wontwork] and [action.auto_create_index] is [false]]")); assertFalse("Operation on existing index should succeed", responses[2].isFailed()); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index 00ca9e8f6e270..16280ce3d96f5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -102,7 +102,7 @@ public void testUsageOfDataStreamFails() throws IOException { Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser)); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null, - null, new ComposableIndexTemplate.DataStreamTemplate()); + null, new ComposableIndexTemplate.DataStreamTemplate(), null); Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet()); Exception actualException = (Exception) e.getCause(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index f4c8c1749015c..c449594380ea3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -23,15 +23,15 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; +import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; @@ -40,6 +40,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -62,17 +63,20 @@ public static final class TransportAction extends TransportMasterNodeAction indices = bulkRequest.requests.stream() - // delete requests should not attempt to create the index (if the index does not - // exists), unless an external versioning is used - .filter(request -> request.opType() != DocWriteRequest.OpType.DELETE - || request.versionType() == VersionType.EXTERNAL - || request.versionType() == VersionType.EXTERNAL_GTE) - .collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2)); - /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create - * that we'll use when we try to run the requests. */ - final Map indicesThatCannotBeCreated = new HashMap<>(); - Set autoCreateIndices = new HashSet<>(); - ClusterState state = clusterService.state(); - for (Map.Entry indexAndFlag : indices.entrySet()) { - boolean shouldAutoCreate; - final String index = indexAndFlag.getKey(); - try { - shouldAutoCreate = shouldAutoCreate(index, state); - } catch (IndexNotFoundException e) { - shouldAutoCreate = false; - indicesThatCannotBeCreated.put(index, e); - } - // We should only auto create if we are not requiring it to be an alias - if (shouldAutoCreate && (indexAndFlag.getValue() == false)) { - autoCreateIndices.add(index); - } + // Attempt to create all the indices that we're going to need during the bulk before we start. + // Step 1: collect all the indices in the request + final Map indices = bulkRequest.requests.stream() + // delete requests should not attempt to create the index (if the index does not + // exists), unless an external versioning is used + .filter(request -> request.opType() != DocWriteRequest.OpType.DELETE + || request.versionType() == VersionType.EXTERNAL + || request.versionType() == VersionType.EXTERNAL_GTE) + .collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2)); + + // Step 2: filter the list of indices to find those that don't currently exist. + final Map indicesThatCannotBeCreated = new HashMap<>(); + Set autoCreateIndices = new HashSet<>(); + ClusterState state = clusterService.state(); + for (Map.Entry indexAndFlag : indices.entrySet()) { + final String index = indexAndFlag.getKey(); + boolean shouldAutoCreate = indexNameExpressionResolver.hasIndexAbstraction(index, state) == false; + // We should only auto create if we are not requiring it to be an alias + if (shouldAutoCreate && (indexAndFlag.getValue() == false)) { + autoCreateIndices.add(index); } - // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back. - if (autoCreateIndices.isEmpty()) { - executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated); - } else { - final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size()); - for (String index : autoCreateIndices) { - createIndex(index, bulkRequest.timeout(), minNodeVersion, - new ActionListener() { - @Override - public void onResponse(CreateIndexResponse result) { - if (counter.decrementAndGet() == 0) { - threadPool.executor(executorName).execute( - () -> executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated)); - } + } + + // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back. + if (autoCreateIndices.isEmpty()) { + executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated); + } else { + final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size()); + for (String index : autoCreateIndices) { + createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener() { + @Override + public void onResponse(CreateIndexResponse result) { + if (counter.decrementAndGet() == 0) { + threadPool.executor(executorName).execute( + () -> executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated)); } + } - @Override - public void onFailure(Exception e) { - if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) { - // fail all requests involving this index, if create didn't work - for (int i = 0; i < bulkRequest.requests.size(); i++) { - DocWriteRequest request = bulkRequest.requests.get(i); - if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) { - bulkRequest.requests.set(i, null); - } + @Override + public void onFailure(Exception e) { + final Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof IndexNotFoundException) { + indicesThatCannotBeCreated.put(index, (IndexNotFoundException) e); + } + else if ((cause instanceof ResourceAlreadyExistsException) == false) { + // fail all requests involving this index, if create didn't work + for (int i = 0; i < bulkRequest.requests.size(); i++) { + DocWriteRequest request = bulkRequest.requests.get(i); + if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) { + bulkRequest.requests.set(i, null); } } - if (counter.decrementAndGet() == 0) { - threadPool.executor(executorName).execute(() -> executeBulk(task, bulkRequest, startTime, - ActionListener.wrap(listener::onResponse, inner -> { + } + if (counter.decrementAndGet() == 0) { + threadPool.executor(executorName).execute(() -> executeBulk(task, bulkRequest, startTime, + ActionListener.wrap(listener::onResponse, inner -> { inner.addSuppressed(e); listener.onFailure(inner); }), responses, indicesThatCannotBeCreated)); - } } - }); - } + } + }); } - } else { - executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap()); } } @@ -357,10 +345,6 @@ boolean isOnlySystem(BulkRequest request, SortedMap in return request.getIndices().stream().allMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName)); } - boolean includesSystem(BulkRequest request, SortedMap indicesLookup, SystemIndices systemIndices) { - return request.getIndices().stream().anyMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName)); - } - private boolean isSystemIndex(SortedMap indicesLookup, SystemIndices systemIndices, String indexName) { final IndexAbstraction abstraction = indicesLookup.get(indexName); if (abstraction != null) { @@ -370,14 +354,6 @@ private boolean isSystemIndex(SortedMap indicesLookup, } } - boolean needToCheck() { - return autoCreateIndex.needToCheck(); - } - - boolean shouldAutoCreate(String index, ClusterState state) { - return autoCreateIndex.shouldAutoCreate(index, state); - } - void createIndex(String index, TimeValue timeout, Version minNodeVersion, diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index ee340a3bf7b92..e09d59ea70b77 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -20,7 +20,10 @@ package org.elasticsearch.action.support; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -41,7 +44,6 @@ * a write operation is about to happen in a non existing index. */ public final class AutoCreateIndex { - public static final Setting AUTO_CREATE_INDEX_SETTING = new Setting<>("action.auto_create_index", "true", AutoCreate::new, Property.NodeScope, Setting.Property.Dynamic); @@ -61,13 +63,6 @@ public AutoCreateIndex(Settings settings, clusterSettings.addSettingsUpdateConsumer(AUTO_CREATE_INDEX_SETTING, this::setAutoCreate); } - /** - * Do we really need to check if an index should be auto created? - */ - public boolean needToCheck() { - return this.autoCreate.autoCreateIndex; - } - /** * Should the index be auto created? * @throws IndexNotFoundException if the index doesn't exist and shouldn't be auto created @@ -82,6 +77,17 @@ public boolean shouldAutoCreate(String index, ClusterState state) { return true; } + // Templates can override the AUTO_CREATE_INDEX_SETTING setting + final ComposableIndexTemplate template = findTemplate(index, state.metadata()); + if (template != null && template.getAllowAutoCreate() != null) { + if (template.getAllowAutoCreate()) { + return true; + } else { + // An explicit false value overrides AUTO_CREATE_INDEX_SETTING + throw new IndexNotFoundException("composable template " + template.indexPatterns() + " forbids index auto creation"); + } + } + // One volatile read, so that all checks are done against the same instance: final AutoCreate autoCreate = this.autoCreate; if (autoCreate.autoCreateIndex == false) { @@ -118,6 +124,11 @@ void setAutoCreate(AutoCreate autoCreate) { this.autoCreate = autoCreate; } + private ComposableIndexTemplate findTemplate(String indexName, Metadata metadata) { + final String templateName = MetadataIndexTemplateService.findV2Template(metadata, indexName, false); + return metadata.templatesV2().get(templateName); + } + static class AutoCreate { private final boolean autoCreateIndex; private final List> expressions; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index aae931ba3661e..2c38039493d19 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -56,6 +56,9 @@ public class ComposableIndexTemplate extends AbstractDiffable PARSER = new ConstructingObjectParser<>("index_template", @@ -66,7 +69,8 @@ public class ComposableIndexTemplate extends AbstractDiffable) a[5], - (DataStreamTemplate) a[6])); + (DataStreamTemplate) a[6], + (Boolean) a[7])); static { PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDEX_PATTERNS); @@ -76,6 +80,7 @@ public class ComposableIndexTemplate extends AbstractDiffable p.map(), METADATA); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DataStreamTemplate.PARSER, DATA_STREAM); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ALLOW_AUTO_CREATE); } private final List indexPatterns; @@ -91,6 +96,8 @@ public class ComposableIndexTemplate extends AbstractDiffable metadata; @Nullable private final DataStreamTemplate dataStreamTemplate; + @Nullable + private final Boolean allowAutoCreate; static Diff readITV2DiffFrom(StreamInput in) throws IOException { return AbstractDiffable.readDiffFrom(ComposableIndexTemplate::new, in); @@ -102,12 +109,12 @@ public static ComposableIndexTemplate parse(XContentParser parser) throws IOExce public ComposableIndexTemplate(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, @Nullable Long priority, @Nullable Long version, @Nullable Map metadata) { - this(indexPatterns, template, componentTemplates, priority, version, metadata, null); + this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null); } public ComposableIndexTemplate(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, @Nullable Long priority, @Nullable Long version, @Nullable Map metadata, - @Nullable DataStreamTemplate dataStreamTemplate) { + @Nullable DataStreamTemplate dataStreamTemplate, @Nullable Boolean allowAutoCreate) { this.indexPatterns = indexPatterns; this.template = template; this.componentTemplates = componentTemplates; @@ -115,6 +122,7 @@ public ComposableIndexTemplate(List indexPatterns, @Nullable Template te this.version = version; this.metadata = metadata; this.dataStreamTemplate = dataStreamTemplate; + this.allowAutoCreate = allowAutoCreate; } public ComposableIndexTemplate(StreamInput in) throws IOException { @@ -133,6 +141,11 @@ public ComposableIndexTemplate(StreamInput in) throws IOException { } else { this.dataStreamTemplate = null; } + if (in.getVersion().onOrAfter(ALLOW_AUTO_CREATE_VERSION)) { + this.allowAutoCreate = in.readOptionalBoolean(); + } else { + this.allowAutoCreate = null; + } } public List indexPatterns() { @@ -174,6 +187,11 @@ public DataStreamTemplate getDataStreamTemplate() { return dataStreamTemplate; } + @Nullable + public Boolean getAllowAutoCreate() { + return this.allowAutoCreate; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(this.indexPatterns); @@ -190,6 +208,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_9_0)) { out.writeOptionalWriteable(dataStreamTemplate); } + if (out.getVersion().onOrAfter(ALLOW_AUTO_CREATE_VERSION)) { + out.writeOptionalBoolean(allowAutoCreate); + } } @Override @@ -214,6 +235,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (this.dataStreamTemplate != null) { builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate); } + if (this.allowAutoCreate != null) { + builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate); + } builder.endObject(); return builder; } @@ -221,7 +245,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public int hashCode() { return Objects.hash(this.indexPatterns, this.template, this.componentTemplates, this.priority, this.version, - this.metadata, this.dataStreamTemplate); + this.metadata, this.dataStreamTemplate, this.allowAutoCreate); } @Override @@ -239,7 +263,8 @@ public boolean equals(Object obj) { Objects.equals(this.priority, other.priority) && Objects.equals(this.version, other.version) && Objects.equals(this.metadata, other.metadata) && - Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate); + Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate) && + Objects.equals(this.allowAutoCreate, other.allowAutoCreate); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 1a16b663e5ef4..18b5346753f1a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -476,7 +476,8 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo final Template finalTemplate = new Template(finalSettings, stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases()); finalIndexTemplate = new ComposableIndexTemplate(template.indexPatterns(), finalTemplate, template.composedOf(), - template.priority(), template.version(), template.metadata(), template.getDataStreamTemplate()); + template.priority(), template.version(), template.metadata(), template.getDataStreamTemplate(), + template.getAllowAutoCreate()); } if (finalIndexTemplate.equals(existing)) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java index 7cdd8561c8294..ea8435790a5c1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java @@ -32,37 +32,41 @@ public class AutoCreateActionTests extends ESTestCase { - public void testResolveAutoCreateDataStreams() { + public void testResolveTemplates() { Metadata metadata; { Metadata.Builder mdBuilder = new Metadata.Builder(); DataStreamTemplate dataStreamTemplate = new DataStreamTemplate(); - mdBuilder.put("1", new ComposableIndexTemplate(Collections.singletonList("legacy-logs-*"), null, null, 10L, null, null, null)); + mdBuilder.put("1", new ComposableIndexTemplate(Collections.singletonList("legacy-logs-*"), + null, null, 10L, null, null, null, null)); mdBuilder.put("2", new ComposableIndexTemplate(Collections.singletonList("logs-*"), - null, null, 20L, null, null, dataStreamTemplate)); + null, null, 20L, null, null, dataStreamTemplate, null)); mdBuilder.put("3", new ComposableIndexTemplate(Collections.singletonList("logs-foobar"), - null, null, 30L, null, null, dataStreamTemplate)); + null, null, 30L, null, null, dataStreamTemplate, null)); metadata = mdBuilder.build(); } CreateIndexRequest request = new CreateIndexRequest("logs-foobar"); - DataStreamTemplate result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + ComposableIndexTemplate result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, notNullValue()); - assertThat(result.getTimestampField(), equalTo("@timestamp")); + assertThat(result.getDataStreamTemplate(), notNullValue()); + assertThat(result.getDataStreamTemplate().getTimestampField(), equalTo("@timestamp")); request = new CreateIndexRequest("logs-barbaz"); - result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, notNullValue()); - assertThat(result.getTimestampField(), equalTo("@timestamp")); + assertThat(result.getDataStreamTemplate(), notNullValue()); + assertThat(result.getDataStreamTemplate().getTimestampField(), equalTo("@timestamp")); // An index that matches with a template without a data steam definition request = new CreateIndexRequest("legacy-logs-foobaz"); - result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); - assertThat(result, nullValue()); + result = AutoCreateAction.resolveTemplate(request, metadata); + assertThat(result, notNullValue()); + assertThat(result.getDataStreamTemplate(), nullValue()); // An index that doesn't match with an index template request = new CreateIndexRequest("my-index"); - result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, nullValue()); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index e1bac1919b6c4..fb23ee3ebf7a5 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -376,7 +376,7 @@ public void testRejectDuplicateAliasV2() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("foo-*", "bar-*"), new Template(null, null, aliases), - null, null, null, null, null); + null, null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("test-template", template).build(); @@ -393,7 +393,7 @@ public void testRejectDuplicateAliasV2UsingComponentTemplates() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null); final ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("foo-*", "bar-*"), null, - Collections.singletonList("ct"), null, null, null, null); + Collections.singletonList("ct"), null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("ct", ct) @@ -429,7 +429,7 @@ public void testHiddenAffectsResolvedV2Templates() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("*"), new Template(null, null, aliases), - null, null, null, null, null); + null, null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("test-template", template).build(); @@ -450,7 +450,7 @@ public void testHiddenAffectsResolvedV2ComponentTemplates() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null); final ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("*"), null, - Collections.singletonList("ct"), null, null, null, null); + Collections.singletonList("ct"), null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("ct", ct) @@ -542,7 +542,7 @@ public void testRolloverClusterState() throws Exception { public void testRolloverClusterStateForDataStream() throws Exception { final DataStream dataStream = DataStreamTestHelper.randomInstance(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"), - null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); Metadata.Builder builder = Metadata.builder(); builder.put("template", template); for (Index index : dataStream.getIndices()) { @@ -641,7 +641,7 @@ public void testValidation() throws Exception { sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName(); defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"), - null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); builder.put("template", template); for (Index index : dataStream.getIndices()) { builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java index b56bcaf4df48a..b0da6388f7017 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -65,7 +65,7 @@ public void testIndexNameCannotBeNullOrEmpty() { public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(Collections.singletonList("*"), - template, null, null, null, null, null); + template, null, null, null, null, null, null); PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java index a2b8bef24fe1e..ac783342a0238 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java @@ -66,7 +66,7 @@ public void testIndexNameCannotBeNullOrEmpty() { public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(Collections.singletonList("*"), - template, null, null, null, null, null); + template, null, null, null, null, null, null); PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java index 75a29feff2f4e..cc62c4ede2038 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java @@ -59,7 +59,7 @@ protected PutComposableIndexTemplateAction.Request mutateInstance(PutComposableI public void testPutGlobalTemplatesCannotHaveHiddenIndexSetting() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), - template, null, null, null, null, null); + template, null, null, null, null, null, null); PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); @@ -85,7 +85,7 @@ public void testPutIndexTemplateV2RequestMustContainTemplate() { public void testValidationOfPriority() { PutComposableIndexTemplateAction.Request req = new PutComposableIndexTemplateAction.Request("test"); - req.indexTemplate(new ComposableIndexTemplate(Arrays.asList("foo", "bar"), null, null, -5L, null, null, null)); + req.indexTemplate(new ComposableIndexTemplate(Arrays.asList("foo", "bar"), null, null, -5L, null, null, null, null)); ActionRequestValidationException validationException = req.validate(); assertThat(validationException, is(notNullValue())); List validationErrors = validationException.validationErrors(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index f4ef6176fde30..ce0bc8ba5269a 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -35,6 +36,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; @@ -45,35 +47,35 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.Arrays; -import java.util.HashSet; +import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; -import java.util.function.Predicate; +import java.util.function.Consumer; +import java.util.function.Function; -import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; +import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class TransportBulkActionIndicesThatCannotBeCreatedTests extends ESTestCase { + private static final Consumer noop = index -> {}; + public void testNonExceptional() { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); bulkRequest.add(new DeleteRequest(randomAlphaOfLength(5))); bulkRequest.add(new UpdateRequest(randomAlphaOfLength(5), randomAlphaOfLength(5), randomAlphaOfLength(5))); - // Test emulating auto_create_index=false - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, null); - // Test emulating auto_create_index=true - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true); - // Test emulating all indices already created - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false); + // Test emulating that index can be auto-created + indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true, noop); + // Test emulating that index cannot be auto-created + indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false, noop); // Test emulating auto_create_index=true with some indices already created. - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean()); + indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean(), noop); } public void testAllFail() { @@ -82,9 +84,12 @@ public void testAllFail() { bulkRequest.add(new IndexRequest("can't")); bulkRequest.add(new DeleteRequest("do").version(0).versionType(VersionType.EXTERNAL)); bulkRequest.add(new UpdateRequest("nothin", randomAlphaOfLength(5), randomAlphaOfLength(5))); - indicesThatCannotBeCreatedTestCase(new HashSet<>(Arrays.asList("no", "can't", "do", "nothin")), bulkRequest, index -> { - throw new IndexNotFoundException("Can't make it because I say so"); - }); + indicesThatCannotBeCreatedTestCase( + newHashSet("no", "can't", "do", "nothin"), + bulkRequest, + index -> true, + index -> { throw new IndexNotFoundException("Can't make it because I say so"); } + ); } public void testSomeFail() { @@ -92,40 +97,44 @@ public void testSomeFail() { bulkRequest.add(new IndexRequest("ok")); bulkRequest.add(new IndexRequest("bad")); // Emulate auto_create_index=-bad,+* - indicesThatCannotBeCreatedTestCase(singleton("bad"), bulkRequest, index -> { - if (index.equals("bad")) { - throw new IndexNotFoundException("Can't make it because I say so"); - } - return true; - }); - // Emulate auto_create_index=false but the "ok" index already exists - indicesThatCannotBeCreatedTestCase(singleton("bad"), bulkRequest, index -> { + indicesThatCannotBeCreatedTestCase(singleton("bad"), bulkRequest, index -> true, index -> { if (index.equals("bad")) { throw new IndexNotFoundException("Can't make it because I say so"); } - return false; }); } private void indicesThatCannotBeCreatedTestCase(Set expected, - BulkRequest bulkRequest, Predicate shouldAutoCreate) { + BulkRequest bulkRequest, Function shouldAutoCreate, Consumer simulateAutoCreate) { ClusterService clusterService = mock(ClusterService.class); ClusterState state = mock(ClusterState.class); when(state.getMetadata()).thenReturn(Metadata.EMPTY_METADATA); when(state.metadata()).thenReturn(Metadata.EMPTY_METADATA); when(clusterService.state()).thenReturn(state); + DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); when(state.getNodes()).thenReturn(discoveryNodes); when(discoveryNodes.getMinNodeVersion()).thenReturn(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT)); + DiscoveryNode localNode = mock(DiscoveryNode.class); when(clusterService.localNode()).thenReturn(localNode); when(localNode.isIngestNode()).thenReturn(randomBoolean()); + final ThreadPool threadPool = mock(ThreadPool.class); final ExecutorService direct = EsExecutors.newDirectExecutorService(); when(threadPool.executor(anyString())).thenReturn(direct); + + final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)) { + @Override + public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) { + return shouldAutoCreate.apply(indexAbstraction) == false; + } + }; + TransportBulkAction action = new TransportBulkAction(threadPool, mock(TransportService.class), clusterService, - null, null, null, mock(ActionFilters.class), null, null, new IndexingPressure(Settings.EMPTY), new SystemIndices(emptyMap())) { + null, null, null, mock(ActionFilters.class), indexNameExpressionResolver, + new IndexingPressure(Settings.EMPTY), new SystemIndices(Collections.emptyMap())) { @Override void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener listener, AtomicArray responses, Map indicesThatCannotBeCreated) { @@ -133,21 +142,25 @@ void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, Action } @Override - boolean needToCheck() { - return null != shouldAutoCreate; // Use "null" to mean "no indices can be created so don't bother checking" + void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener listener) { + try { + simulateAutoCreate.accept(index); + // If we try to create an index just immediately assume it worked + listener.onResponse(new CreateIndexResponse(true, true, index) { + }); + } catch (Exception e) { + listener.onFailure(e); + } } - + }; + action.doExecute(null, bulkRequest, new ActionListener() { @Override - boolean shouldAutoCreate(String index, ClusterState state) { - return shouldAutoCreate.test(index); - } + public void onResponse(BulkResponse bulkItemResponses) {} @Override - void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener listener) { - // If we try to create an index just immediately assume it worked - listener.onResponse(new CreateIndexResponse(true, true, index) {}); + public void onFailure(Exception e) { + throw new AssertionError(e); } - }; - action.doExecute(null, bulkRequest, null); + }); } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 3fd8c34ca25d4..867df6341c4fa 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -34,10 +34,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -46,7 +46,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; @@ -145,18 +144,11 @@ class TestTransportBulkAction extends TransportBulkAction { TestTransportBulkAction() { super(threadPool, transportService, clusterService, ingestService, - null, null, new ActionFilters(Collections.emptySet()), null, - new AutoCreateIndex( - SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), - new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), - new SystemIndices(emptyMap()) - ), new IndexingPressure(SETTINGS), new SystemIndices(emptyMap()) + null, null, new ActionFilters(Collections.emptySet()), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), + new IndexingPressure(SETTINGS), new SystemIndices(emptyMap()) ); } - @Override - protected boolean needToCheck() { - return needToCheck; - } + @Override void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos, final ActionListener listener, final AtomicArray responses, Map indicesThatCannotBeCreated) { @@ -637,7 +629,7 @@ public void testFindDefaultPipelineFromV2TemplateMatch() { ComposableIndexTemplate t1 = new ComposableIndexTemplate(Collections.singletonList("missing_*"), new Template(Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "pipeline2").build(), null, null), - null, null, null, null, null); + null, null, null, null, null, null); ClusterState state = clusterService.state(); Metadata metadata = Metadata.builder() diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 3e77001892d91..8ed420a94d5e1 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterState; @@ -88,15 +87,9 @@ class TestTransportBulkAction extends TransportBulkAction { TestTransportBulkAction() { super(TransportBulkActionTests.this.threadPool, transportService, clusterService, null, null, null, new ActionFilters(Collections.emptySet()), new Resolver(), - new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver(), new SystemIndices(emptyMap())), new IndexingPressure(Settings.EMPTY), new SystemIndices(emptyMap())); } - @Override - protected boolean needToCheck() { - return true; - } - @Override void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener listener) { indexCreated = true; @@ -271,28 +264,6 @@ public void testOnlySystem() { assertFalse(bulkAction.isOnlySystem(buildBulkRequest(mixed), indicesLookup, systemIndices)); } - public void testIncludesSystem() { - SortedMap indicesLookup = new TreeMap<>(); - Settings settings = Settings.builder().put("index.version.created", Version.CURRENT).build(); - indicesLookup.put(".foo", - new Index(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); - indicesLookup.put(".bar", - new Index(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); - SystemIndices systemIndices = new SystemIndices(org.elasticsearch.common.collect.Map.of("plugin", - org.elasticsearch.common.collect.List.of(new SystemIndexDescriptor(".test", "")))); - List onlySystem = org.elasticsearch.common.collect.List.of(".foo", ".bar"); - assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices)); - - onlySystem = org.elasticsearch.common.collect.List.of(".foo", ".bar", ".test"); - assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices)); - - List nonSystem = org.elasticsearch.common.collect.List.of("foo", "bar"); - assertFalse(bulkAction.includesSystem(buildBulkRequest(nonSystem), indicesLookup, systemIndices)); - - List mixed = org.elasticsearch.common.collect.List.of(".foo", ".test", "other"); - assertTrue(bulkAction.includesSystem(buildBulkRequest(mixed), indicesLookup, systemIndices)); - } - private BulkRequest buildBulkRequest(List indices) { BulkRequest request = new BulkRequest(); for (String index : indices) { diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 054436a3a88fa..7507504ff9394 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -129,7 +128,6 @@ void doExecute(ActionType action, Request request, ActionListener + autoCreateIndex.shouldAutoCreate(randomIndex, clusterState)); + assertEquals("no such index [" + randomIndex + "] and [action.auto_create_index] is [false]", e.getMessage()); + } + + /** + * Check that if a template matches the index to be created, but that template has the allow_auto_create + * setting turned off, then it overrides the global setting. + */ + public void testDisabledAutoCreateTemplateSettingDoesNotOverride() { + String randomIndex = randomAlphaOfLengthBetween(2, 10); + final ComposableIndexTemplate template = new ComposableIndexTemplate( + singletonList(randomIndex.charAt(0) + "*"), + null, + emptyList(), + null, + null, + emptyMap(), + null, + false + ); + + final Metadata metadata = Metadata.builder().indexTemplates(singletonMap("test_template", template)).build(); + final ClusterState clusterState = ClusterState.builder(buildClusterState()).metadata(metadata).build(); + + Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build(); + AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings); + IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> + autoCreateIndex.shouldAutoCreate(randomIndex, clusterState)); + assertEquals("no such index [composable template [" + randomIndex.charAt(0) + "*] forbids index auto creation]", e.getMessage()); + } + + /** + * Check that if a template matches the index to be created, and that template has the allow_auto_create + * setting enabled, then it overrides the global setting. + */ + public void testEnabledAutoCreateTemplateSettingDoesOverride() { + String randomIndex = randomAlphaOfLengthBetween(2, 10); + final ComposableIndexTemplate template = new ComposableIndexTemplate( + singletonList(randomIndex.charAt(0) + "*"), + null, + emptyList(), + null, + null, + emptyMap(), + null, + true + ); + + final Metadata metadata = Metadata.builder().indexTemplates(singletonMap("test_template", template)).build(); + final ClusterState clusterState = ClusterState.builder(buildClusterState()).metadata(metadata).build(); + + Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build(); + AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings); + assertThat(autoCreateIndex.shouldAutoCreate(randomIndex, clusterState), equalTo(true)); + } + private static ClusterState buildClusterState(String... indices) { Metadata.Builder metadata = Metadata.builder(); for (String index : indices) { @@ -221,8 +333,8 @@ private static ClusterState buildClusterState(String... indices) { } private AutoCreateIndex newAutoCreateIndex(Settings settings) { - SystemIndices systemIndices = new SystemIndices(org.elasticsearch.common.collect.Map.of("plugin", - org.elasticsearch.common.collect.List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, "")))); + SystemIndices systemIndices = new SystemIndices(singletonMap("plugin", + singletonList(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, "")))); return new AutoCreateIndex(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), systemIndices); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index 24efa12b3c966..10d805f780502 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -97,7 +97,9 @@ public static ComposableIndexTemplate randomInstance() { randomBoolean() ? null : randomNonNegativeLong(), randomBoolean() ? null : randomNonNegativeLong(), meta, - dataStreamTemplate); + dataStreamTemplate, + randomBoolean() ? null : randomBoolean() + ); } private static Map randomAliases() { @@ -163,7 +165,7 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori List newIndexPatterns = randomValueOtherThan(orig.indexPatterns(), () -> randomList(1, 4, () -> randomAlphaOfLength(4))); return new ComposableIndexTemplate(newIndexPatterns, orig.template(), orig.composedOf(), - orig.priority(), orig.version(), orig.metadata(), orig.getDataStreamTemplate()); + orig.priority(), orig.version(), orig.metadata(), orig.getDataStreamTemplate(), null); case 1: return new ComposableIndexTemplate(orig.indexPatterns(), randomValueOtherThan(orig.template(), () -> new Template(randomSettings(), @@ -172,7 +174,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 2: List newComposedOf = randomValueOtherThan(orig.composedOf(), () -> randomList(0, 10, () -> randomAlphaOfLength(5))); @@ -182,7 +185,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 3: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -190,7 +194,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori randomValueOtherThan(orig.priority(), ESTestCase::randomNonNegativeLong), orig.version(), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 4: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -198,7 +203,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 5: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -206,7 +212,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), randomValueOtherThan(orig.metadata(), ComposableIndexTemplateTests::randomMeta), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 6: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -214,7 +221,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), orig.metadata(), - randomValueOtherThan(orig.getDataStreamTemplate(), ComposableIndexTemplateTests::randomDataStreamTemplate)); + randomValueOtherThan(orig.getDataStreamTemplate(), ComposableIndexTemplateTests::randomDataStreamTemplate), + orig.getAllowAutoCreate()); default: throw new IllegalStateException("illegal randomization branch"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index a6d2b4bc9c000..2c72871668391 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -47,7 +47,7 @@ public void testCreateDataStream() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"), - null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); @@ -126,7 +126,7 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; ComposableIndexTemplate template = - new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"), null, null, null, null, null, null); + new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"), null, null, null, null, null, null, null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); @@ -141,7 +141,7 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { public static ClusterState createDataStream(final String dataStreamName) throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"), - null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 6454ffef8fd7a..8510a696522c5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -389,17 +389,17 @@ public void testUpdateComponentTemplateWithIndexHiddenSetting() throws Exception ComposableIndexTemplate firstGlobalIndexTemplate = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), template, - org.elasticsearch.common.collect.List.of("foo"), 1L, null, null, null); + org.elasticsearch.common.collect.List.of("foo"), 1L, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate1", firstGlobalIndexTemplate); ComposableIndexTemplate secondGlobalIndexTemplate = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), template, - org.elasticsearch.common.collect.List.of("foo"), 2L, null, null, null); + org.elasticsearch.common.collect.List.of("foo"), 2L, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate2", secondGlobalIndexTemplate); ComposableIndexTemplate fooPatternIndexTemplate = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("foo-*"), template, - org.elasticsearch.common.collect.List.of("foo"), 3L, null, null, null); + org.elasticsearch.common.collect.List.of("foo"), 3L, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "foopatternindextemplate", fooPatternIndexTemplate); // update the component template to set the index.hidden setting @@ -451,7 +451,7 @@ public void testUpdateIndexTemplateV2() throws Exception { List patterns = new ArrayList<>(template.indexPatterns()); patterns.add("new-pattern"); template = new ComposableIndexTemplate(patterns, template.template(), template.composedOf(), template.priority(), - template.version(), template.metadata(), null); + template.version(), template.metadata(), null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template); assertNotNull(state.metadata().templatesV2().get("foo")); @@ -489,7 +489,7 @@ public void testPuttingV2TemplateGeneratesWarning() throws Exception { .build(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -526,7 +526,7 @@ public void onFailure(Exception e) { waitToCreateComponentTemplate.await(10, TimeUnit.SECONDS); ComposableIndexTemplate globalIndexTemplate = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), - null, org.elasticsearch.common.collect.List.of("ct-with-index-hidden-setting"), null, null, null, null); + null, org.elasticsearch.common.collect.List.of("ct-with-index-hidden-setting"), null, null, null, null, null); expectThrows(InvalidIndexTemplateException.class, () -> metadataIndexTemplateService.putIndexTemplateV2("testing", true, "template-referencing-ct-with-hidden-index-setting", @@ -551,7 +551,7 @@ public void onFailure(Exception e) { public void testPuttingV1StarTemplateGeneratesWarning() throws Exception { final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); @@ -572,7 +572,7 @@ public void testPuttingV1StarTemplateGeneratesWarning() throws Exception { public void testPuttingV1NonStarTemplateGeneratesWarning() throws Exception { final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); @@ -602,7 +602,7 @@ public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() .build(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -643,7 +643,7 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesWarning() t .build(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -668,11 +668,11 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesWarning() t public void testPuttingOverlappingV2Template() throws Exception { { ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("egg*", "baz"), - null, null, 1L, null, null, null); + null, null, 1L, null, null, null, null); MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Arrays.asList("abc", "baz*"), - null, null, 1L, null, null, null); + null, null, 1L, null, null, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo2", newTemplate)); assertThat(e.getMessage(), equalTo("index template [foo2] has index patterns [abc, baz*] matching patterns from existing " + @@ -682,11 +682,11 @@ public void testPuttingOverlappingV2Template() throws Exception { { ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("egg*", "baz"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Arrays.asList("abc", "baz*"), - null, null, 0L, null, null, null); + null, null, 0L, null, null, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo2", newTemplate)); assertThat(e.getMessage(), equalTo("index template [foo2] has index patterns [abc, baz*] matching patterns from existing " + @@ -703,10 +703,10 @@ public void testFindV2Templates() throws Exception { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); ComposableIndexTemplate it = new ComposableIndexTemplate(Collections.singletonList("i*"), - null, Collections.singletonList("ct"), null, 1L, null, null); + null, Collections.singletonList("ct"), null, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); ComposableIndexTemplate it2 = new ComposableIndexTemplate(Collections.singletonList("in*"), - null, Collections.singletonList("ct"), 10L, 2L, null, null); + null, Collections.singletonList("ct"), 10L, 2L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean()); @@ -722,10 +722,10 @@ public void testFindV2TemplatesForHiddenIndex() throws Exception { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); ComposableIndexTemplate it = new ComposableIndexTemplate(Collections.singletonList("i*"), - null, Collections.singletonList("ct"), 0L, 1L, null, null); + null, Collections.singletonList("ct"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); ComposableIndexTemplate it2 = new ComposableIndexTemplate(Collections.singletonList("*"), - null, Collections.singletonList("ct"), 10L, 2L, null, null); + null, Collections.singletonList("ct"), 10L, 2L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true); @@ -738,7 +738,7 @@ public void testFindV2InvalidGlobalTemplate() { try { // add an invalid global template that specifies the `index.hidden` setting ComposableIndexTemplate invalidGlobalTemplate = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("*"), - templateWithHiddenSetting, org.elasticsearch.common.collect.List.of("ct"), 5L, 1L, null, null); + templateWithHiddenSetting, org.elasticsearch.common.collect.List.of("ct"), 5L, 1L, null, null, null); Metadata invalidGlobalTemplateMetadata = Metadata.builder().putCustom(ComposableIndexTemplateMetadata.TYPE, new ComposableIndexTemplateMetadata(org.elasticsearch.common.collect.Map.of("invalid_global_template", invalidGlobalTemplate))).build(); @@ -782,7 +782,7 @@ public void testResolveConflictingMappings() throws Exception { " }\n" + " }\n" + " }"), null), - Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null); + Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); @@ -848,7 +848,7 @@ public void testResolveMappings() throws Exception { " }\n" + " }\n" + " }"), null), - Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null); + Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); @@ -903,7 +903,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex " }\n" + " }"), null), org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, true, "logs-data-stream-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "logs-data-stream-template", @@ -955,7 +955,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex " }\n" + " }\n" + " }"), null), - org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null, null); + org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries"); @@ -1041,7 +1041,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception state = service.addComponentTemplate(state, true, "ct1", ct1); ComposableIndexTemplate it = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("logs*"), null, org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, true, "logs-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "logs-template", @@ -1086,7 +1086,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception " }"), null); ComposableIndexTemplate it = new ComposableIndexTemplate( org.elasticsearch.common.collect.List.of("timeseries*"), template, null, 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", @@ -1141,7 +1141,7 @@ public void testResolveSettings() throws Exception { .put("index.blocks.write", false) .put("index.number_of_shards", 3) .build(), null, null), - Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null); + Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), "my-template"); @@ -1169,7 +1169,7 @@ public void testResolveAliases() throws Exception { state = service.addComponentTemplate(state, true, "ct_low", ct2); ComposableIndexTemplate it = new ComposableIndexTemplate(Collections.singletonList("i*"), new Template(null, null, a3), - Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null); + Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(state.metadata(), "my-template"); @@ -1285,7 +1285,7 @@ public void testIndexTemplateFailsToOverrideComponentTemplateMappingField() thr " }\n" + " }\n" + " }"), null), - randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null); + randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null, null); final ClusterState finalState = state; IllegalArgumentException e = expectThrows(IllegalArgumentException.class, @@ -1336,7 +1336,7 @@ public void testUpdateComponentTemplateFailsIfResolvedIndexTemplatesWouldBeInval state = service.addComponentTemplate(state, true, "c2", ct2); ComposableIndexTemplate it = new ComposableIndexTemplate(Arrays.asList("i*"), new Template(null, null, null), - randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null); + randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null, null); // Great, the templates aren't invalid state = service.addIndexTemplateV2(state, randomBoolean(), "my-template", it); @@ -1408,7 +1408,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { .build(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"), - null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, false, "logs", template); @@ -1430,7 +1430,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Test replacing it with a version without the data stream config IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"), null, null, - 100L, null, null, null); + 100L, null, null, null, null); service.addIndexTemplateV2(stateWithDS, false, "logs", nonDSTemplate); }); @@ -1441,7 +1441,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Test adding a higher priority version that would cause problems e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-my*-*"), null, null, - 105L, null, null, null); + 105L, null, null, null, null); service.addIndexTemplateV2(stateWithDS, false, "logs2", nonDSTemplate); }); @@ -1452,7 +1452,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Change the pattern to one that doesn't match the data stream e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null, - null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); service.addIndexTemplateV2(stateWithDS, false, "logs", newTemplate); }); @@ -1462,12 +1462,12 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Add an additional template that matches our data stream at a lower priority ComposableIndexTemplate mysqlTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-mysql-*"), null, - null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); ClusterState stateWithDSAndTemplate = service.addIndexTemplateV2(stateWithDS, false, "logs-mysql", mysqlTemplate); // We should be able to replace the "logs" template, because we have the "logs-mysql" template that can handle the data stream ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null, null, - 100L, null, null, null); + 100L, null, null, null, null); service.addIndexTemplateV2(stateWithDSAndTemplate, false, "logs", nonDSTemplate); } @@ -1551,9 +1551,9 @@ clusterService, createIndexService, new AliasValidator(), indicesService, @SuppressWarnings("unchecked") public static void assertTemplatesEqual(ComposableIndexTemplate actual, ComposableIndexTemplate expected) { ComposableIndexTemplate actualNoTemplate = new ComposableIndexTemplate(actual.indexPatterns(), null, - actual.composedOf(), actual.priority(), actual.version(), actual.metadata(), actual.getDataStreamTemplate()); + actual.composedOf(), actual.priority(), actual.version(), actual.metadata(), actual.getDataStreamTemplate(), null); ComposableIndexTemplate expectedNoTemplate = new ComposableIndexTemplate(expected.indexPatterns(), null, - expected.composedOf(), expected.priority(), expected.version(), expected.metadata(), expected.getDataStreamTemplate()); + expected.composedOf(), expected.priority(), expected.version(), expected.metadata(), expected.getDataStreamTemplate(), null); assertThat(actualNoTemplate, equalTo(expectedNoTemplate)); Template actualTemplate = actual.template(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index d2954d26e6668..bb4dc637cf832 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -78,7 +78,8 @@ public void testSimpleJsonFromAndTo() throws IOException { 5L, 4L, Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken")), - randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate())) + randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate(), + null)) .put(IndexMetadata.builder("test12") .settings(settings(Version.CURRENT) .put("setting1", "value1") diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 1c7ae17f095ea..fef04cbee742e 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -83,7 +83,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.PlainActionFuture; @@ -1573,7 +1572,7 @@ public void onFailure(final Exception e) { shardStateAction, actionFilters), RetentionLeaseSyncer.EMPTY); - Map actions = new HashMap<>(); + Map actions = new HashMap<>(); final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService); final MetadataCreateIndexService metadataCreateIndexService = new MetadataCreateIndexService(settings, clusterService, indicesService, @@ -1587,9 +1586,9 @@ allocationService, new AliasValidator(), shardLimitValidator, environment, index )); final MappingUpdatedAction mappingUpdatedAction = new MappingUpdatedAction(settings, clusterSettings, clusterService); mappingUpdatedAction.setClient(client); - final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction(settings, transportService, - clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction, new UpdateHelper(scriptService), - actionFilters, new IndexingPressure(settings), new SystemIndices(emptyMap())); + final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction(settings, transportService, + clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction, new UpdateHelper(scriptService), + actionFilters, new IndexingPressure(settings), new SystemIndices(emptyMap())); actions.put(BulkAction.INSTANCE, new TransportBulkAction(threadPool, transportService, clusterService, new IngestService( @@ -1597,7 +1596,6 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi new AnalysisModule(environment, Collections.emptyList()).getAnalysisRegistry(), Collections.emptyList(), client), transportShardBulkAction, client, actionFilters, indexNameExpressionResolver, - new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, new SystemIndices(emptyMap())), new IndexingPressure(settings), new SystemIndices(emptyMap()) )); diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java index 4a2e2de4cdc7b..856d64f38e9aa 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java @@ -172,7 +172,7 @@ public void testTemplateOverridesDefaults() { Template t = new Template(Settings.builder() .put(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE, DataTier.DATA_WARM) .build(), null, null); - ComposableIndexTemplate ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null); + ComposableIndexTemplate ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null, null); client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); @@ -189,7 +189,7 @@ public void testTemplateOverridesDefaults() { t = new Template(Settings.builder() .putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER) .build(), null, null); - ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null); + ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null, null); client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index e3f80404e23bb..994e996510b11 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -38,7 +38,8 @@ public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry { // version 2: converted to hidden index // version 3: templates moved to composable templates // version 4:converted data stream - public static final int INDEX_TEMPLATE_VERSION = 4; + // version 5: add `allow_auto_create` setting + public static final int INDEX_TEMPLATE_VERSION = 5; public static final String SLM_TEMPLATE_VERSION_VARIABLE = "xpack.slm.template.version"; public static final String SLM_TEMPLATE_NAME = ".slm-history"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java index b71e12852ab2d..d9f58efac77fa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java @@ -17,20 +17,26 @@ public final class WatcherIndexTemplateRegistryField { // version 10: add support for foreach path in actions // version 11: watch history indices are hidden // version 12: templates changed to composable templates + // version 13: add `allow_auto_create` setting // Note: if you change this, also inform the kibana team around the watcher-ui - public static final int INDEX_TEMPLATE_VERSION = 12; + public static final int INDEX_TEMPLATE_VERSION = 13; public static final int INDEX_TEMPLATE_VERSION_10 = 10; public static final int INDEX_TEMPLATE_VERSION_11 = 11; + public static final int INDEX_TEMPLATE_VERSION_12 = 12; public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION; public static final String HISTORY_TEMPLATE_NAME_10 = ".watch-history-" + INDEX_TEMPLATE_VERSION_10; public static final String HISTORY_TEMPLATE_NAME_11 = ".watch-history-" + INDEX_TEMPLATE_VERSION_11; + public static final String HISTORY_TEMPLATE_NAME_12 = ".watch-history-" + INDEX_TEMPLATE_VERSION_12; public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; - public static final String HISTORY_TEMPLATE_NAME_NO_ILM_10 = ".watch-history-no-ilm-10"; + public static final String HISTORY_TEMPLATE_NAME_NO_ILM_10 = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION_10; public static final String HISTORY_TEMPLATE_NAME_NO_ILM_11 = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION_11; + public static final String HISTORY_TEMPLATE_NAME_NO_ILM_12 = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION_12; public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches"; public static final String TRIGGERED_TEMPLATE_NAME_11 = ".triggered_watches-11"; + public static final String TRIGGERED_TEMPLATE_NAME_12 = ".triggered_watches-12"; public static final String WATCHES_TEMPLATE_NAME = ".watches"; public static final String WATCHES_TEMPLATE_NAME_11 = ".watches-11"; + public static final String WATCHES_TEMPLATE_NAME_12 = ".watches-12"; public static final String[] TEMPLATE_NAMES = new String[] { HISTORY_TEMPLATE_NAME, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME }; diff --git a/x-pack/plugin/core/src/main/resources/ilm-history.json b/x-pack/plugin/core/src/main/resources/ilm-history.json index 8fa18c742c277..44ad66d9dabdc 100644 --- a/x-pack/plugin/core/src/main/resources/ilm-history.json +++ b/x-pack/plugin/core/src/main/resources/ilm-history.json @@ -80,6 +80,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for ILM history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/logs-template.json b/x-pack/plugin/core/src/main/resources/logs-template.json index 0e23872ec0170..9c21a2b6ae504 100644 --- a/x-pack/plugin/core/src/main/resources/logs-template.json +++ b/x-pack/plugin/core/src/main/resources/logs-template.json @@ -6,6 +6,7 @@ "logs-mappings", "logs-settings" ], + "allow_auto_create": true, "_meta": { "description": "default logs template installed by x-pack", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/logstash-management.json b/x-pack/plugin/core/src/main/resources/logstash-management.json index d9528238dc0fb..00b0826a02574 100644 --- a/x-pack/plugin/core/src/main/resources/logstash-management.json +++ b/x-pack/plugin/core/src/main/resources/logstash-management.json @@ -7,6 +7,7 @@ "codec": "best_compression" } }, + "allow_auto_create": true, "mappings" : { "_doc" : { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/metrics-template.json b/x-pack/plugin/core/src/main/resources/metrics-template.json index cccd852196d41..0da4169de79b7 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-template.json +++ b/x-pack/plugin/core/src/main/resources/metrics-template.json @@ -6,6 +6,7 @@ "metrics-mappings", "metrics-settings" ], + "allow_auto_create": true, "_meta": { "description": "default metrics template installed by x-pack", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/slm-history.json b/x-pack/plugin/core/src/main/resources/slm-history.json index ab6699e0758be..974e72e0795e9 100644 --- a/x-pack/plugin/core/src/main/resources/slm-history.json +++ b/x-pack/plugin/core/src/main/resources/slm-history.json @@ -57,6 +57,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for SLM history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/synthetics-template.json b/x-pack/plugin/core/src/main/resources/synthetics-template.json index 9bbbe0ef9e3df..1036a29cf5a9d 100644 --- a/x-pack/plugin/core/src/main/resources/synthetics-template.json +++ b/x-pack/plugin/core/src/main/resources/synthetics-template.json @@ -6,6 +6,7 @@ "synthetics-mappings", "synthetics-settings" ], + "allow_auto_create": true, "_meta": { "description": "default synthetics template installed by x-pack", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/triggered-watches.json b/x-pack/plugin/core/src/main/resources/triggered-watches.json index 1cb43c51c6ef5..e11f7d9853186 100644 --- a/x-pack/plugin/core/src/main/resources/triggered-watches.json +++ b/x-pack/plugin/core/src/main/resources/triggered-watches.json @@ -37,6 +37,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for triggered watches indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json index 2f4433536d4ae..d9de434e08275 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json +++ b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json @@ -613,6 +613,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for watcher history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/watch-history.json b/x-pack/plugin/core/src/main/resources/watch-history.json index c5b380165fd78..7ee688da832d0 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history.json +++ b/x-pack/plugin/core/src/main/resources/watch-history.json @@ -564,6 +564,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for watcher history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/watches.json b/x-pack/plugin/core/src/main/resources/watches.json index ab9ba4b1d9ec6..4c1108c9f38ad 100644 --- a/x-pack/plugin/core/src/main/resources/watches.json +++ b/x-pack/plugin/core/src/main/resources/watches.json @@ -59,6 +59,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for watches indices", "managed": true diff --git a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java new file mode 100644 index 0000000000000..83dbf56a0a26b --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.xpack.datastreams; + +import org.elasticsearch.action.support.AutoCreateIndex; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.time.Instant; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.Matchers.containsString; + +public class AutoCreateDataStreamIT extends ESRestTestCase { + + /** + * Check that setting {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} to false + * does not affect the ability to auto-create data streams, which are not subject to the setting. + */ + public void testCanAutoCreateDataStreamWhenAutoCreateIndexDisabled() throws IOException { + configureAutoCreateIndex(false); + createTemplateWithAllowAutoCreate(null); + assertOK(this.indexDocument()); + } + + /** + * Check that automatically creating a data stream is allowed when the index name matches a template + * and that template has allow_auto_create set to true. + */ + public void testCanAutoCreateDataStreamWhenExplicitlyAllowedByTemplate() throws IOException { + createTemplateWithAllowAutoCreate(true); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + assertOK(this.indexDocument()); + } + + /** + * Check that automatically creating a data stream is disallowed when the data stream name matches a template and that template has + * allow_auto_create explicitly to false. + */ + public void testCannotAutoCreateDataStreamWhenDisallowedByTemplate() throws IOException { + createTemplateWithAllowAutoCreate(false); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), + containsString("no such index [composable template [recipe*] forbids index auto creation]") + ); + } + + private void configureAutoCreateIndex(boolean value) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("transient") + .field(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), value) + .endObject() + .endObject(); + + final Request settingsRequest = new Request("PUT", "_cluster/settings"); + settingsRequest.setJsonEntity(Strings.toString(builder)); + final Response settingsResponse = client().performRequest(settingsRequest); + assertOK(settingsResponse); + } + + private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws IOException { + XContentBuilder b = JsonXContent.contentBuilder(); + b.startObject(); + { + b.array("index_patterns", "recipe*"); + if (allowAutoCreate != null) { + b.field("allow_auto_create", allowAutoCreate); + } + b.startObject("data_stream"); + b.endObject(); + } + b.endObject(); + + final Request createTemplateRequest = new Request("PUT", "_index_template/recipe_template"); + createTemplateRequest.setJsonEntity(Strings.toString(b)); + final Response createTemplateResponse = client().performRequest(createTemplateRequest); + assertOK(createTemplateResponse); + } + + private Response indexDocument() throws IOException { + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc"); + indexDocumentRequest.setJsonEntity("{ \"@timestamp\": \"" + Instant.now() + "\", \"name\": \"Kimchi\" }"); + return client().performRequest(indexDocumentRequest); + } +} diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index d429543f8ded5..a4f81a4a3cde6 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -429,7 +429,8 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); @@ -512,7 +513,8 @@ public void testTimeStampValidationInvalidFieldMapping() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); @@ -974,7 +976,8 @@ public void testMixedAutoCreate() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet(); @@ -1231,7 +1234,8 @@ static void putComposableIndexTemplate( null, null, metadata, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java index 02451c147b085..f2e24458c264b 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java @@ -44,7 +44,8 @@ public void testDefaultDataStreamAllocateToHot() { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ); client().execute( PutComposableIndexTemplateAction.INSTANCE, diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java index e2dac9060ec44..e352f4bc3a718 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java @@ -218,7 +218,8 @@ private String createDataStream() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ); assertTrue( client().execute( diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index 08494b7fb70e8..c8835acbd0b39 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -29,7 +29,8 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry { // version 1: initial // version 2: convert to hidden index // version 3: templates moved to composable templates - public static final int INDEX_TEMPLATE_VERSION = 3; + // version 4: add `allow_auto_create` setting + public static final int INDEX_TEMPLATE_VERSION = 4; public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version"; public static final String ILM_TEMPLATE_NAME = "ilm-history"; diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 904b3f9e2bf00..fbe3f3abbb5f7 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -291,7 +291,8 @@ protected static void createDataStreamAndTemplate(String dataStreamName, String null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()))) + new ComposableIndexTemplate.DataStreamTemplate(), + null))) .actionGet(); client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet(); } diff --git a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/90_auto_create_index.yml b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/90_auto_create_index.yml new file mode 100644 index 0000000000000..b9b9a39dab776 --- /dev/null +++ b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/90_auto_create_index.yml @@ -0,0 +1,52 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + - do: + cluster.put_settings: + body: + transient: + action.auto_create_index: false + +--- +teardown: + - do: + watcher.delete_watch: + id: "test_watch" + ignore: 404 + - do: + cluster.put_settings: + body: + transient: + action.auto_create_index: null + +--- +"Ensure watch is created even when auto_create_index is disabled": + - do: + watcher.put_watch: + id: "test_watch" + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "simple": { + "foo": "bar" + } + }, + "condition": { + "always": {} + }, + "actions": { + "indexme" : { + "index" : { + "index" : "my_test_index", + "doc_id": "my-id" + } + } + } + } + - match: { _id: "test_watch" } + - match: { created: true } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java index 01fa7d2c0f994..f5c5041cd7054 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java @@ -11,11 +11,18 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.not; public class WatcherRestartIT extends AbstractUpgradeTestCase { @@ -38,29 +45,90 @@ private void validateHistoryTemplate() throws Exception { // v7.7.0 contains a Watch history template (version 11) that can't be used unless all nodes in the cluster are >=7.7.0, so // in a mixed cluster with some nodes <7.7.0 it will install template version 10, but if all nodes are <=7.7.0 template v11 // is used. - final String expectedFinalTemplate = templatePrefix + "12"; + final String expectedFinalTemplate = templatePrefix + "13"; // In 7.10 watcher templates were converted to composable index templates, so we only // check legacy templates if we upgraded from a version that had legacy templates. - if (UPGRADE_FROM_VERSION.before(Version.V_7_10_0)) { - final String expectedMixedClusterTemplate = templatePrefix + (UPGRADE_FROM_VERSION.before(Version.V_7_7_0) ? "10" : "11"); - if (ClusterType.MIXED == CLUSTER_TYPE) { - final Request request = new Request("HEAD", "/_template/" + expectedMixedClusterTemplate); - request.addParameter("include_type_name", "false"); - RequestOptions.Builder builder = request.getOptions().toBuilder(); - builder.setWarningsHandler(WarningsHandler.PERMISSIVE); - request.setOptions(builder); - Response response = client().performRequest(request); - assertThat(response.getStatusLine().getStatusCode(), is(200)); + // In 7.11 watcher templates were updated to add `allow_auto_create` + if (UPGRADE_FROM_VERSION.before(Version.V_7_11_0)) { + String expectedMixedClusterTemplate = templatePrefix; + if (UPGRADE_FROM_VERSION.before(Version.V_7_7_0)) { + expectedMixedClusterTemplate += "10"; + } else if (UPGRADE_FROM_VERSION.before(Version.V_7_10_0)) { + expectedMixedClusterTemplate += "11"; + } else { + expectedMixedClusterTemplate += "12"; + } + + if (UPGRADE_FROM_VERSION.before(Version.V_7_10_0) && ClusterType.MIXED == CLUSTER_TYPE) { + assertTemplateExists(expectedMixedClusterTemplate); } else if (ClusterType.UPGRADED == CLUSTER_TYPE) { - Response response = client().performRequest(new Request("HEAD", "/_index_template/" + expectedFinalTemplate)); - assertThat(response.getStatusLine().getStatusCode(), is(200)); + assertIndexTemplateExists(expectedFinalTemplate); } } else { - Response response = client().performRequest(new Request("HEAD", "/_index_template/" + expectedFinalTemplate)); - assertThat(response.getStatusLine().getStatusCode(), is(200)); + assertIndexTemplateExists(expectedFinalTemplate); } } + /** + * Checks that the specified template exists. + *

+ * The implementation fetches all templates so that if the template doesn't exist, + * then the error messages can state which templates do exist, for debugging + * purposes. + */ + private void assertTemplateExists(String expectedTemplate) throws IOException { + final Request request = new Request("GET", "/_template"); + request.addParameter("include_type_name", "false"); + RequestOptions.Builder builder = request.getOptions().toBuilder(); + builder.setWarningsHandler(WarningsHandler.PERMISSIVE); + request.setOptions(builder); + Response response = client().performRequest(request); + assertOK(response); + + final Map responseMap = XContentHelper.convertToMap( + JsonXContent.jsonXContent, + response.getEntity().getContent(), + true + ); + + final Set templateNames = responseMap.keySet(); + + assertThat( + "For version " + UPGRADE_FROM_VERSION + ", template " + expectedTemplate + " not found in: " + templateNames, + templateNames, + hasItem(expectedTemplate) + ); + } + + /** + * Checks that the specified index template exists. + *

+ * The implementation fetches all templates so that if the template doesn't exist, + * then the error messages can state which templates do exist, for debugging + * purposes. + */ + @SuppressWarnings("unchecked") + private void assertIndexTemplateExists(String expectedTemplate) throws IOException { + final Response response = client().performRequest(new Request("GET", "/_index_template")); + assertOK(response); + + final Map responseMap = XContentHelper.convertToMap( + JsonXContent.jsonXContent, + response.getEntity().getContent(), + true + ); + + List> templates = (List>) responseMap.get("index_templates"); + + final List templateNames = templates.stream().map(each -> (String) each.get("name")).collect(Collectors.toList()); + + assertThat( + "For version " + UPGRADE_FROM_VERSION + ", template " + expectedTemplate + " not found in: " + templateNames, + templateNames, + hasItem(expectedTemplate) + ); + } + private void ensureWatcherStopped() throws Exception { assertBusy(() -> { Response stats = client().performRequest(new Request("GET", "_watcher/stats"));