From e69c83533c33eba67fe4a65b3ff03d890494591c Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 27 Aug 2020 12:51:12 -0600 Subject: [PATCH] Allocate newly created indices on data_hot tier nodes (#61342) This commit adds the functionality to allocate newly created indices on nodes in the "hot" tier by default when they are created. This does not break existing behavior, as nodes with the `data` role are considered to be part of the hot tier. Users that separate their deployments by using the `data_hot` (and `data_warm`, `data_cold`, `data_frozen`) roles will have their data allocated on the hot tier nodes now by default. This change is a little more complicated than changing the default value for `index.routing.allocation.include._tier` from null to "data_hot". Instead, this adds the ability to have a plugin inject a setting into the builder for a newly created index. This has the benefit of allowing this setting to be visible as part of the settings when retrieving the index, for example: ``` // Create an index PUT /eggplant // Get an index GET /eggplant?flat_settings ``` Returns the default settings now of: ```json { "eggplant" : { "aliases" : { }, "mappings" : { }, "settings" : { "index.creation_date" : "1597855465598", "index.number_of_replicas" : "1", "index.number_of_shards" : "1", "index.provided_name" : "eggplant", "index.routing.allocation.include._tier" : "data_hot", "index.uuid" : "6ySG78s9RWGystRipoBFCA", "index.version.created" : "8000099" } } } ``` After the initial setting of this setting, it can be treated like any other index level setting. This new setting is *not* set on a new index if any of the following is true: - The index is created with an `index.routing.allocation.include.` setting - The index is created with an `index.routing.allocation.exclude.` setting - The index is created with an `index.routing.allocation.require.` setting - The index is created with a null `index.routing.allocation.include._tier` value - The index was created from an existing source metadata (shrink, clone, split, etc) Relates to #60848 --- docs/reference/api-conventions.asciidoc | 8 + .../cluster/allocation-explain.asciidoc | 7 +- .../elasticsearch/upgrades/RecoveryIT.java | 3 + .../metadata/MetadataCreateIndexService.java | 82 ++++++- .../cluster/node/DiscoveryNodeFilters.java | 3 + .../index/shard/IndexSettingProvider.java | 36 +++ .../java/org/elasticsearch/node/Node.java | 3 + .../org/elasticsearch/plugins/Plugin.java | 11 + .../MetadataCreateIndexServiceTests.java | 25 +- .../ccr/PrimaryFollowerAllocationIT.java | 6 +- .../elasticsearch/xpack/core/DataTier.java | 34 +++ .../elasticsearch/xpack/core/XPackPlugin.java | 6 + .../routing/allocation/DataTierIT.java | 219 ++++++++++++++++++ .../core/LocalStateCompositeXPackPlugin.java | 18 +- .../ilm/TimeSeriesLifecycleActionsIT.java | 7 +- 15 files changed, 444 insertions(+), 24 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/shard/IndexSettingProvider.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 68ffd82eda383..48855ffa4ffa7 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -391,6 +391,7 @@ Returns: "index.creation_date": "1474389951325", "index.uuid": "n6gzFZTgS664GUfx0Xrpjw", "index.version.created": ..., + "index.routing.allocation.include._tier" : "data_hot", "index.provided_name" : "my-index-000001" } } @@ -424,6 +425,13 @@ Returns: "version": { "created": ... }, + "routing": { + "allocation": { + "include": { + "_tier": "data_hot" + } + } + }, "provided_name" : "my-index-000001" } } diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc index 2ea27332c397b..1a238d7713607 100644 --- a/docs/reference/cluster/allocation-explain.asciidoc +++ b/docs/reference/cluster/allocation-explain.asciidoc @@ -113,7 +113,12 @@ DELETE my-index-000001 [source,console] -------------------------------------------------- PUT /my-index-000001?master_timeout=1s&timeout=1s -{"settings": {"index.routing.allocation.include._name": "non_existent_node"} } +{ + "settings": { + "index.routing.allocation.include._name": "non_existent_node", + "index.routing.allocation.include._tier": null + } +} GET /_cluster/allocation/explain { diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index a1829ff3f5248..f698183d60a7b 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -250,6 +250,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception { // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN // before timing out .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") + .put("index.routing.allocation.include._tier", "") .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster createIndex(index, settings.build()); indexDocs(index, 0, 10); @@ -266,6 +267,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception { .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) .put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null) .put("index.routing.allocation.include._id", oldNode) + .putNull("index.routing.allocation.include._tier") ); ensureGreen(index); // wait for the primary to be assigned ensureNoInitializingShards(); // wait for all other shard activity to finish @@ -288,6 +290,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception { updateIndexSettings(index, Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2) .put("index.routing.allocation.include._id", (String)null) + .putNull("index.routing.allocation.include._tier") ); asyncIndexDocs(index, 60, 45).get(); ensureGreen(index); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index babaf38ba0a02..08c59798c4db6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -69,6 +69,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.shard.IndexSettingProvider; import org.elasticsearch.indices.IndexCreationException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexNameException; @@ -85,6 +86,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -130,6 +132,7 @@ public class MetadataCreateIndexService { private final SystemIndices systemIndices; private final ShardLimitValidator shardLimitValidator; private final boolean forbidPrivateIndexSettings; + private final Set indexSettingProviders = new HashSet<>(); public MetadataCreateIndexService( final Settings settings, @@ -158,6 +161,19 @@ public MetadataCreateIndexService( this.shardLimitValidator = shardLimitValidator; } + /** + * Add a provider to be invoked to get additional index settings prior to an index being created + */ + public void addAdditionalIndexSettingProvider(IndexSettingProvider provider) { + if (provider == null) { + throw new IllegalArgumentException("provider must not be null"); + } + if (indexSettingProviders.contains(provider)) { + throw new IllegalArgumentException("provider already added"); + } + this.indexSettingProviders.add(provider); + } + /** * Validate the name for an index against some static rules and a cluster state. */ @@ -471,7 +487,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(templates), - null, settings, indexScopedSettings, shardLimitValidator); + null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -497,7 +513,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName), - null, settings, indexScopedSettings, shardLimitValidator); + null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -548,7 +564,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt } final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, Settings.EMPTY, - sourceMetadata, settings, indexScopedSettings, shardLimitValidator); + sourceMetadata, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -635,14 +651,64 @@ static Map> parseV1Mappings(Map requ * @return the aggregated settings for the new index */ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClusterStateUpdateRequest request, - Settings templateSettings, @Nullable IndexMetadata sourceMetadata, Settings settings, - IndexScopedSettings indexScopedSettings, ShardLimitValidator shardLimitValidator) { - Settings.Builder indexSettingsBuilder = Settings.builder(); + Settings combinedTemplateSettings, @Nullable IndexMetadata sourceMetadata, Settings settings, + IndexScopedSettings indexScopedSettings, ShardLimitValidator shardLimitValidator, + Set indexSettingProviders) { + // Create builders for the template and request settings. We transform these into builders + // because we may want settings to be "removed" from these prior to being set on the new + // index (see more comments below) + final Settings.Builder templateSettings = Settings.builder().put(combinedTemplateSettings); + final Settings.Builder requestSettings = Settings.builder().put(request.settings()); + + final Settings.Builder indexSettingsBuilder = Settings.builder(); if (sourceMetadata == null) { - indexSettingsBuilder.put(templateSettings); + final Settings.Builder additionalIndexSettings = Settings.builder(); + final Settings templateAndRequestSettings = Settings.builder() + .put(combinedTemplateSettings) + .put(request.settings()) + .build(); + + // Loop through all the explicit index setting providers, adding them to the + // additionalIndexSettings map + for (IndexSettingProvider provider : indexSettingProviders) { + additionalIndexSettings.put(provider.getAdditionalIndexSettings(request.index(), templateAndRequestSettings)); + } + + // For all the explicit settings, we go through the template and request level settings + // and see if either a template or the request has "cancelled out" an explicit default + // setting. For example, if a plugin had as an explicit setting: + // "index.mysetting": "blah + // And either a template or create index request had: + // "index.mysetting": null + // We want to remove the explicit setting not only from the explicitly set settings, but + // also from the template and request settings, so that from the newly create index's + // perspective it is as though the setting has not been set at all (using the default + // value). + for (String explicitSetting : additionalIndexSettings.keys()) { + if (templateSettings.keys().contains(explicitSetting) && templateSettings.get(explicitSetting) == null) { + logger.debug("removing default [{}] setting as it in set to null in a template for [{}] creation", + explicitSetting, request.index()); + additionalIndexSettings.remove(explicitSetting); + templateSettings.remove(explicitSetting); + } + if (requestSettings.keys().contains(explicitSetting) && requestSettings.get(explicitSetting) == null) { + logger.debug("removing default [{}] setting as it in set to null in the request for [{}] creation", + explicitSetting, request.index()); + additionalIndexSettings.remove(explicitSetting); + requestSettings.remove(explicitSetting); + } + } + + // Finally, we actually add the explicit defaults prior to the template settings and the + // request settings, so that the precedence goes: + // Explicit Defaults -> Template -> Request -> Necessary Settings (# of shards, uuid, etc) + indexSettingsBuilder.put(additionalIndexSettings.build()); + indexSettingsBuilder.put(templateSettings.build()); } + // now, put the request settings, so they override templates - indexSettingsBuilder.put(request.settings()); + indexSettingsBuilder.put(requestSettings.build()); + if (indexSettingsBuilder.get(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey()) == null) { final DiscoveryNodes nodes = currentState.nodes(); final Version createdVersion = Version.min(Version.CURRENT, nodes.getSmallestNonClientNodeVersion()); diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java index aacda43864e51..ee9f124306cff 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java @@ -181,6 +181,9 @@ public boolean match(DiscoveryNode node) { } } } + } else if ("_tier".equals(attr)) { + // Always allow _tier as an attribute, will be handled elsewhere + return true; } else { String nodeAttributeValue = node.getAttributes().get(attr); if (nodeAttributeValue == null) { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/shard/IndexSettingProvider.java new file mode 100644 index 0000000000000..ce074666aed48 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexSettingProvider.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.settings.Settings; + +/** + * An {@link IndexSettingProvider} is a provider for index level settings that can be set + * explicitly as a default value (so they show up as "set" for newly created indices) + */ +public interface IndexSettingProvider { + /** + * Returns explicitly set default index {@link Settings} for the given index. This should not + * return null. + */ + default Settings getAdditionalIndexSettings(String indexName, Settings templateAndRequestSettings) { + return Settings.EMPTY; + } +} diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 95bafe9ef57a0..593b80517851d 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -520,6 +520,9 @@ protected Node(final Environment initialEnvironment, systemIndices, forbidPrivateIndexSettings ); + pluginsService.filterPlugins(Plugin.class) + .forEach(p -> p.getAdditionalIndexSettingProviders() + .forEach(metadataCreateIndexService::addAdditionalIndexSettingProvider)); final MetadataCreateDataStreamService metadataCreateDataStreamService = new MetadataCreateDataStreamService(threadPool, clusterService, metadataCreateIndexService); diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index 51abe3fcb3f12..e8cf4e2fd4389 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -39,6 +39,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.shard.IndexSettingProvider; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ExecutorBuilder; @@ -227,4 +228,14 @@ public Set getRoles() { public void close() throws IOException { } + + /** + * An {@link IndexSettingProvider} allows hooking in to parts of an index + * lifecycle to provide explicit default settings for newly created indices. Rather than changing + * the default values for an index-level setting, these act as though the setting has been set + * explicitly, but still allow the setting to be overridden by a template or creation request body. + */ + public Collection getAdditionalIndexSettingProviders() { + return Collections.emptyList(); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 2e7ead49e08fa..aa9c7ab9b2c90 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -609,7 +609,8 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() { request.settings(Settings.builder().put("request_setting", "value2").build()); Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService()); + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), + Collections.emptySet()); assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1")); assertThat(aggregatedIndexSettings.get("request_setting"), equalTo("value2")); @@ -646,7 +647,8 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { MetadataIndexTemplateService.resolveAliases(Collections.singletonList(templateMetadata)), Metadata.builder().build(), aliasValidator, xContentRegistry(), queryShardContext); Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templateMetadata.settings(), - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService()); + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), + Collections.emptySet()); assertThat(resolvedAliases.get(0).getSearchRouting(), equalTo("fromRequest")); assertThat(aggregatedIndexSettings.get("key1"), equalTo("requestValue")); @@ -662,7 +664,8 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { public void testDefaultSettings() { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService()); + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), + Collections.emptySet()); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("1")); } @@ -670,7 +673,7 @@ public void testDefaultSettings() { public void testSettingsFromClusterState() { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService()); + randomShardLimitService(), Collections.emptySet()); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("15")); } @@ -694,7 +697,7 @@ public void testTemplateOrder() throws Exception { )); Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, MetadataIndexTemplateService.resolveSettings(templates), null, Settings.EMPTY, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService()); + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), Collections.emptySet()); List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), MetadataIndexTemplateService.resolveAliases(templates), Metadata.builder().build(), aliasValidator, xContentRegistry(), queryShardContext); @@ -722,7 +725,7 @@ public void testAggregateIndexSettingsIgnoresTemplatesOnCreateFromSourceIndex() Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), clusterState.metadata().index("sourceIndex"), Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService()); + randomShardLimitService(), Collections.emptySet()); assertThat(aggregatedIndexSettings.get("templateSetting"), is(nullValue())); assertThat(aggregatedIndexSettings.get("requestSetting"), is("requestValue")); @@ -909,12 +912,12 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() { assertThat(targetRoutingNumberOfShards, is(6)); } - public void testSoftDeletesDisabledDeprecation() { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService()); + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), + Collections.emptySet()); assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " + "Please do not specify value for setting [index.soft_deletes.enabled] of index [test]."); request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); @@ -922,7 +925,8 @@ public void testSoftDeletesDisabledDeprecation() { request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build()); } aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService()); + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), + Collections.emptySet()); } public void testValidateTranslogRetentionSettings() { @@ -935,7 +939,8 @@ public void testValidateTranslogRetentionSettings() { } request.settings(settings.build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService()); + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), + Collections.emptySet()); assertWarnings("Translog retention settings [index.translog.retention.age] " + "and [index.translog.retention.size] are deprecated and effectively ignored. They will be removed in a future version."); } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java index 3d9b4e23e3879..a5dc15680df26 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java @@ -50,6 +50,7 @@ public void testDoNotAllocateFollowerPrimaryToNodesWithoutRemoteClusterClientRol final PutFollowAction.Request putFollowRequest = putFollow(leaderIndex, followerIndex); putFollowRequest.setSettings(Settings.builder() .put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes)) + .putNull("index.routing.allocation.include._tier") .build()); putFollowRequest.waitForActiveShards(ActiveShardCount.ONE); putFollowRequest.timeout(TimeValue.timeValueSeconds(2)); @@ -83,6 +84,7 @@ public void testAllocateFollowerPrimaryToNodesWithRemoteClusterClientRole() thro .put("index.routing.rebalance.enable", "none") .put("index.routing.allocation.include._name", Stream.concat(dataOnlyNodes.stream(), dataAndRemoteNodes.stream()).collect(Collectors.joining(","))) + .putNull("index.routing.allocation.include._tier") .build()); final PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, putFollowRequest).get(); assertTrue(response.isFollowIndexShardsAcked()); @@ -105,7 +107,9 @@ public void testAllocateFollowerPrimaryToNodesWithRemoteClusterClientRole() thro // Follower primaries can be relocated to nodes without the remote cluster client role followerClient().admin().indices().prepareUpdateSettings(followerIndex) .setMasterNodeTimeout(TimeValue.MAX_VALUE) - .setSettings(Settings.builder().put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes))) + .setSettings(Settings.builder() + .put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes)) + .putNull("index.routing.allocation.include._tier")) .get(); assertBusy(() -> { final ClusterState state = getFollowerCluster().client().admin().cluster().prepareState().get().getState(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java index 08e093aa0ff51..e0c876e561a86 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java @@ -6,10 +6,17 @@ package org.elasticsearch.xpack.core; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexSettingProvider; +import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; + +import java.util.Set; /** * The {@code DataTier} class encapsulates the formalization of the "hot", @@ -137,4 +144,31 @@ public static boolean isColdNode(DiscoveryNode discoveryNode) { public static boolean isFrozenNode(DiscoveryNode discoveryNode) { return discoveryNode.getRoles().contains(DATA_FROZEN_NODE_ROLE) || discoveryNode.getRoles().contains(DiscoveryNodeRole.DATA_ROLE); } + + /** + * This setting provider injects the setting allocating all newly created indices with + * {@code index.routing.allocation.include._tier: "data_hot"} unless the user overrides the + * setting while the index is being created (in a create index request for instance) + */ + public static class DefaultHotAllocationSettingProvider implements IndexSettingProvider { + private static final Logger logger = LogManager.getLogger(DefaultHotAllocationSettingProvider.class); + + @Override + public Settings getAdditionalIndexSettings(String indexName, Settings indexSettings) { + Set settings = indexSettings.keySet(); + if (settings.contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)) { + // It's okay to put it, it will be removed or overridden by the template/request settings + return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build(); + } else if (settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".")) || + settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + ".")) || + settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "."))) { + // A different index level require, include, or exclude has been specified, so don't put the setting + logger.debug("index [{}] specifies custom index level routing filtering, skipping hot tier allocation", indexName); + return Settings.EMPTY; + } else { + // Otherwise, put the setting in place by default + return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build(); + } + } + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index c12bbd58786bb..0b8ca41d0fe27 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -39,6 +39,7 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.shard.IndexSettingProvider; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.LicensesMetadata; @@ -423,6 +424,11 @@ public Collection createAllocationDeciders(Settings settings, return Collections.singleton(new DataTierAllocationDecider(clusterSettings)); } + @Override + public Collection getAdditionalIndexSettingProviders() { + return Collections.singleton(new DataTier.DefaultHotAllocationSettingProvider()); + } + /** * Handles the creation of the SSLService along with the necessary actions to enable reloading * of SSLContexts when configuration files change on disk. diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java new file mode 100644 index 0000000000000..d20c7169d3cca --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java @@ -0,0 +1,219 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.cluster.routing.allocation; + +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.DataTier; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) +public class DataTierIT extends ESIntegTestCase { + private static final String index = "myindex"; + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(LocalStateCompositeXPackPlugin.class); + } + + public void testDefaultAllocateToHot() { + startWarmOnlyNode(); + startColdOnlyNode(); + ensureGreen(); + + client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get(); + + Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); + assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT)); + + // index should be red + assertThat(client().admin().cluster().prepareHealth(index).get().getIndices().get(index).getStatus(), + equalTo(ClusterHealthStatus.RED)); + + logger.info("--> starting hot node"); + startHotOnlyNode(); + + logger.info("--> waiting for {} to be yellow", index); + ensureYellow(index); + } + + public void testOverrideDefaultAllocation() { + startWarmOnlyNode(); + startColdOnlyNode(); + ensureGreen(); + + String setting = randomBoolean() ? DataTierAllocationDecider.INDEX_ROUTING_REQUIRE : + DataTierAllocationDecider.INDEX_ROUTING_INCLUDE; + + client().admin().indices().prepareCreate(index) + .setWaitForActiveShards(0) + .setSettings(Settings.builder() + .put(setting, DataTier.DATA_WARM)) + .get(); + + Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); + assertThat(idxSettings.get(setting), equalTo(DataTier.DATA_WARM)); + + // index should be yellow + logger.info("--> waiting for {} to be yellow", index); + ensureYellow(index); + } + + public void testRequestSettingOverridesAllocation() { + startWarmOnlyNode(); + startColdOnlyNode(); + ensureGreen(); + + client().admin().indices().prepareCreate(index) + .setWaitForActiveShards(0) + .setSettings(Settings.builder() + .putNull(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)) + .get(); + + Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); + assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo("")); + // Even the key shouldn't exist if it has been nulled out + assertFalse(idxSettings.keySet().toString(), idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)); + + // index should be yellow + logger.info("--> waiting for {} to be yellow", index); + ensureYellow(index); + + client().admin().indices().prepareDelete(index).get(); + + // Now test it overriding the "require" setting, in which case the include should be skipped + client().admin().indices().prepareCreate(index) + .setWaitForActiveShards(0) + .setSettings(Settings.builder() + .put(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE, DataTier.DATA_COLD)) + .get(); + + idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); + assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo("")); + // The key should not be put in place since it was overridden + assertFalse(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)); + assertThat(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE_SETTING.get(idxSettings), equalTo(DataTier.DATA_COLD)); + + // index should be yellow + logger.info("--> waiting for {} to be yellow", index); + ensureYellow(index); + } + + /** + * When a new index is created from source metadata (as during a shrink), the data tier + * default setting should *not* be applied. This test checks that behavior. + */ + public void testShrinkStaysOnTier() { + startWarmOnlyNode(); + startHotOnlyNode(); + + client().admin().indices().prepareCreate(index) + .setWaitForActiveShards(0) + .setSettings(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, "data_warm")) + .get(); + + client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.READ_ONLY, index).get(); + client().admin().indices().prepareResizeIndex(index, index + "-shrunk") + .setResizeType(ResizeType.SHRINK) + .setSettings(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build()).get(); + + ensureGreen(index + "-shrunk"); + + Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index + "-shrunk") + .get().getSettings().get(index + "-shrunk"); + // It should inherit the setting of its originator + assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_WARM)); + + // Required or else the test cleanup fails because it can't delete the indices + client().admin().indices().prepareUpdateSettings(index, index + "-shrunk") + .setSettings(Settings.builder() + .put("index.blocks.read_only", false)) + .get(); + } + + public void testTemplateOverridesDefaults() { + startWarmOnlyNode(); + + Template t = new Template(Settings.builder() + .put(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE, DataTier.DATA_WARM) + .build(), null, null); + ComposableIndexTemplate ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null); + client().execute(PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); + + client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get(); + + Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); + assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE), equalTo(false)); + + // index should be yellow + ensureYellow(index); + + client().admin().indices().prepareDelete(index).get(); + + t = new Template(Settings.builder() + .putNull(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE) + .build(), null, null); + ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null); + client().execute(PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); + + client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get(); + + idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); + assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE), equalTo(false)); + + ensureYellow(index); + } + + public void startHotOnlyNode() { + Settings nodeSettings = Settings.builder() + .putList("node.roles", Arrays.asList("master", "data_hot", "ingest")) + .build(); + internalCluster().startNode(nodeSettings); + } + + public void startWarmOnlyNode() { + Settings nodeSettings = Settings.builder() + .putList("node.roles", Arrays.asList("master", "data_warm", "ingest")) + .build(); + internalCluster().startNode(nodeSettings); + } + + public void startColdOnlyNode() { + Settings nodeSettings = Settings.builder() + .putList("node.roles", Arrays.asList("master", "data_cold", "ingest")) + .build(); + internalCluster().startNode(nodeSettings); + } + + public void startFrozenOnlyNode() { + Settings nodeSettings = Settings.builder() + .putList("node.roles", Arrays.asList("master", "data_frozen", "ingest")) + .build(); + internalCluster().startNode(nodeSettings); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index 032e4c51b8016..538cfe7b5d664 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.shard.IndexSettingProvider; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.recovery.RecoverySettings; @@ -380,9 +381,19 @@ public Map getElectionStrategies() { public Set getRoles() { Set roles = new HashSet<>(); filterPlugins(Plugin.class).stream().forEach(p -> roles.addAll(p.getRoles())); + roles.addAll(super.getRoles()); return roles; } + @Override + public Collection getAdditionalIndexSettingProviders() { + Set providers = new HashSet<>(); + filterPlugins(Plugin.class).stream().forEach(p -> providers.addAll(p.getAdditionalIndexSettingProviders())); + providers.addAll(super.getAdditionalIndexSettingProviders()); + return providers; + + } + @Override public Map> getTokenizers() { Map> tokenizers = new HashMap<>(); @@ -493,10 +504,13 @@ public Collection> ind @Override public Collection createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) { - return filterPlugins(ClusterPlugin.class) + Set deciders = new HashSet<>(); + deciders.addAll(filterPlugins(ClusterPlugin.class) .stream() .flatMap(p -> p.createAllocationDeciders(settings, clusterSettings).stream()) - .collect(Collectors.toList()); + .collect(Collectors.toList())); + deciders.addAll(super.createAllocationDeciders(settings, clusterSettings)); + return deciders; } @Override diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 9f85c8f5103f4..cbe5c6ad08c4d 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; import org.elasticsearch.xpack.core.ilm.AllocateAction; import org.elasticsearch.xpack.core.ilm.DeleteAction; import org.elasticsearch.xpack.core.ilm.ForceMergeAction; @@ -652,8 +653,10 @@ public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception int numShards = 2; int expectedFinalShards = 1; String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; - createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); + createIndexWithSettings(client(), index, alias, Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .putNull(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)); ensureGreen(index);