From c567ae795751221d6a11a6a997fb7c04f4e3f544 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Sun, 5 Jan 2020 18:30:17 -0500 Subject: [PATCH 1/4] Deprecate indices without soft-deletes (#50502) Soft-deletes will be enabled for all indices in 8.0. Hence, we should deprecate new indices without soft-deletes in 7.x. --- .../index-modules/history-retention.asciidoc | 3 ++ .../elasticsearch/upgrades/RecoveryIT.java | 31 ++++++++++++--- .../test/indices.create/10_basic.yml | 17 ++++++++ .../test/indices.stats/20_translog.yml | 17 +++++++- .../metadata/MetaDataCreateIndexService.java | 8 ++++ .../MetaDataCreateIndexServiceTests.java | 15 +++++++ .../test/rest/ESRestTestCase.java | 39 ++++++++++++++----- 7 files changed, 113 insertions(+), 17 deletions(-) diff --git a/docs/reference/index-modules/history-retention.asciidoc b/docs/reference/index-modules/history-retention.asciidoc index 94e17e49251e3..8df79625c88d0 100644 --- a/docs/reference/index-modules/history-retention.asciidoc +++ b/docs/reference/index-modules/history-retention.asciidoc @@ -65,6 +65,9 @@ there>>. {ccr-cap} will not function if soft deletes are disabled. configured at index creation and only on indices created on or after 6.5.0. The default value is `true`. + deprecated::[7.6, Creating indices with soft-deletes disabled is + deprecated and will be removed in future Elasticsearch versions.] + `index.soft_deletes.retention_lease.period`:: The maximum length of time to retain a shard history retention lease before diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index f2bd9cbed62f8..a35b1441346d8 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -306,7 +307,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception { } } - public void testRecoveryWithSoftDeletes() throws Exception { + public void testRecovery() throws Exception { final String index = "recover_with_soft_deletes"; if (CLUSTER_TYPE == ClusterType.OLD) { Settings.Builder settings = Settings.builder() @@ -318,8 +319,8 @@ public void testRecoveryWithSoftDeletes() throws Exception { // before timing out .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster - if (getNodeId(v -> v.onOrAfter(Version.V_6_5_0)) != null && randomBoolean()) { - settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); } createIndex(index, settings.build()); int numDocs = randomInt(10); @@ -350,7 +351,7 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); createIndex(index, settings.build()); int numDocs = randomInt(10); indexDocs(index, 0, numDocs); @@ -373,7 +374,7 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1)) .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); createIndex(index, settings.build()); int numDocs = randomInt(10); indexDocs(index, 0, numDocs); @@ -784,6 +785,26 @@ public void testAutoExpandIndicesDuringRollingUpgrade() throws Exception { } } + public void testSoftDeletesDisabledWarning() throws Exception { + final String indexName = "test_soft_deletes_disabled_warning"; + if (CLUSTER_TYPE == ClusterType.OLD) { + boolean softDeletesEnabled = true; + Settings.Builder settings = Settings.builder(); + if (randomBoolean()) { + softDeletesEnabled = randomBoolean(); + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled); + } + Request request = new Request("PUT", "/" + indexName); + request.setJsonEntity("{\"settings\": " + Strings.toString(settings.build()) + "}"); + if (softDeletesEnabled == false) { + expectSoftDeletesWarning(request, indexName); + } + client().performRequest(request); + } + ensureGreen(indexName); + indexDocs(indexName, randomInt(100), randomInt(100)); + } + @SuppressWarnings("unchecked") private Map getIndexSettingsAsMap(String index) throws IOException { Map indexSettings = getIndexSettings(index); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml index 965083421cbaf..cab825734f68e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -154,3 +154,20 @@ - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." } + +--- +"Create index without soft deletes": + - skip: + version: " - 7.5.99" + reason: "indices without soft deletes are deprecated in 7.6" + features: "warnings" + + - do: + warnings: + - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. + Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index]. + indices.create: + index: test_index + body: + settings: + soft_deletes.enabled: false diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml index 69acf85defadc..39c739d79fd8e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml @@ -1,11 +1,19 @@ --- "Translog retention without soft_deletes": + - skip: + version: " - 7.5.99" + reason: "indices without soft deletes are deprecated in 7.6" + features: "warnings" + - do: indices.create: index: test body: settings: soft_deletes.enabled: false + warnings: + - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. + Please do not specify value for setting [index.soft_deletes.enabled] of index [test]. - do: cluster.health: wait_for_no_initializing_shards: true @@ -132,8 +140,9 @@ --- "Translog stats on closed indices without soft-deletes": - skip: - version: " - 7.2.99" - reason: "closed indices have translog stats starting version 7.3.0" + version: " - 7.5.99" + reason: "indices without soft deletes are deprecated in 7.6" + features: "warnings" - do: indices.create: @@ -141,6 +150,10 @@ body: settings: soft_deletes.enabled: false + warnings: + - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. + Please do not specify value for setting [index.soft_deletes.enabled] of index [test]. + - do: cluster.health: wait_for_no_initializing_shards: true diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 5cebacc7b5f03..fc7d9fd17efb1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -53,6 +53,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -63,6 +64,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -104,6 +106,7 @@ */ public class MetaDataCreateIndexService { private static final Logger logger = LogManager.getLogger(MetaDataCreateIndexService.class); + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); public static final int MAX_INDEX_NAME_BYTES = 255; @@ -455,6 +458,11 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu * that will be used to create this index. */ MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState); + if (indexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) { + DEPRECATION_LOGGER.deprecatedAndMaybeLog("soft_deletes_disabled", + "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " + + "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + request.index() + "]."); + } return indexSettings; } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 9afa50ddca6de..e249dddc4390a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -858,6 +858,21 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() { assertThat(targetRoutingNumberOfShards, is(6)); } + public void testSoftDeletesDisabledDeprecation() { + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " + + "Please do not specify value for setting [index.soft_deletes.enabled] of index [test]."); + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + if (randomBoolean()) { + request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build()); + } + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + } + private IndexTemplateMetaData addMatchingTemplate(Consumer configurator) { IndexTemplateMetaData.Builder builder = templateMetaDataBuilder("template1", "te*"); configurator.accept(builder); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index f821a5d1b6082..a1dd9084f5c29 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -52,6 +52,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.SnapshotState; @@ -969,23 +970,27 @@ protected static void ensureNoInitializingShards() throws IOException { } protected static void createIndex(String name, Settings settings) throws IOException { - Request request = new Request("PUT", "/" + name); - request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings) + "}"); - client().performRequest(request); + createIndex(name, settings, null); } protected static void createIndex(String name, Settings settings, String mapping) throws IOException { - Request request = new Request("PUT", "/" + name); - request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings) - + ", \"mappings\" : {" + mapping + "} }"); - client().performRequest(request); + createIndex(name, settings, mapping, null); } protected static void createIndex(String name, Settings settings, String mapping, String aliases) throws IOException { Request request = new Request("PUT", "/" + name); - request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings) - + ", \"mappings\" : {" + mapping + "}" - + ", \"aliases\": {" + aliases + "} }"); + String entity = "{\"settings\": " + Strings.toString(settings); + if (mapping != null) { + entity += ",\"mappings\" : {" + mapping + "}"; + } + if (aliases != null) { + entity += ",\"aliases\": {" + aliases + "}"; + } + entity += "}"; + if (settings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) { + expectSoftDeletesWarning(request, name); + } + request.setJsonEntity(entity); client().performRequest(request); } @@ -1004,6 +1009,20 @@ private static void updateIndexSettings(String index, Settings settings) throws client().performRequest(request); } + protected static void expectSoftDeletesWarning(Request request, String indexName) { + final List expectedWarnings = List.of( + "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " + + "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + indexName + "]."); + final Builder requestOptions = RequestOptions.DEFAULT.toBuilder(); + if (nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_7_6_0))) { + requestOptions.setWarningsHandler(warnings -> warnings.equals(expectedWarnings) == false); + request.setOptions(requestOptions); + } else if (nodeVersions.stream().anyMatch(version -> version.onOrAfter(Version.V_7_6_0))) { + requestOptions.setWarningsHandler(warnings -> warnings.isEmpty() == false && warnings.equals(expectedWarnings) == false); + request.setOptions(requestOptions); + } + } + protected static Map getIndexSettings(String index) throws IOException { Request request = new Request("GET", "/" + index + "/_settings"); request.addParameter("flat_settings", "true"); From 8ddfd03e3e664c6312b530af905fc8db8f474e1e Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Sun, 5 Jan 2020 19:12:04 -0500 Subject: [PATCH 2/4] old collection method --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index a1dd9084f5c29..13bf5b4265a51 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1010,7 +1010,7 @@ private static void updateIndexSettings(String index, Settings settings) throws } protected static void expectSoftDeletesWarning(Request request, String indexName) { - final List expectedWarnings = List.of( + final List expectedWarnings = Collections.singletonList( "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " + "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + indexName + "]."); final Builder requestOptions = RequestOptions.DEFAULT.toBuilder(); From 8d80719a7a3e877f4a0e90ca6031b460d7078fdb Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Sun, 5 Jan 2020 21:26:18 -0500 Subject: [PATCH 3/4] more --- .../cluster/metadata/MetaDataCreateIndexServiceTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index e249dddc4390a..c487c90ae4a1c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -861,7 +861,7 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() { public void testSoftDeletesDisabledDeprecation() { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Collections.emptyList(), Collections.emptyMap(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " + "Please do not specify value for setting [index.soft_deletes.enabled] of index [test]."); @@ -869,7 +869,7 @@ public void testSoftDeletesDisabledDeprecation() { if (randomBoolean()) { request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build()); } - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Collections.emptyList(), Collections.emptyMap(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); } From 124f710667b848b7a8f5b6a851b511e98b67770a Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Sun, 5 Jan 2020 22:35:13 -0500 Subject: [PATCH 4/4] avoid number of shards warning --- .../src/test/java/org/elasticsearch/upgrades/RecoveryIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index a35b1441346d8..3a7ee4e1dbabc 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -789,7 +789,7 @@ public void testSoftDeletesDisabledWarning() throws Exception { final String indexName = "test_soft_deletes_disabled_warning"; if (CLUSTER_TYPE == ClusterType.OLD) { boolean softDeletesEnabled = true; - Settings.Builder settings = Settings.builder(); + Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1); if (randomBoolean()) { softDeletesEnabled = randomBoolean(); settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled);