Make soft-deletes mandatory in 8.0 (#51122)

Creating indices with soft deletes disabled is no longer supported in 8.0.

Author: dnhatn

docs/reference/migration/migrate_8_0/indices.asciidoc

@@ -34,3 +34,12 @@ removed in 8.0.
 
 Synced flush was deprecated in 7.6 and is removed in 8.0. Use a regular flush
 instead as it has the same effect as a synced flush in 7.6 and later.
+
+
+[float]
+==== Indices with soft deletes disabled
+
+Creating indices with soft deletes disabled was deprecated in 7.6 and
+is no longer supported in 8.0. The setting index.soft_deletes.enabled
+can no longer be set to false. As the setting defaults to true, simply
+leave the setting unset.

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -776,6 +776,11 @@ public void testSnapshotRestore() throws IOException {
         if (isRunningAgainstOldCluster()) {
             // Create the index
             count = between(200, 300);
+            Settings.Builder settings = Settings.builder();
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+            }
+            createIndex(index, settings.build());
             indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject());
         } else {
             count = countOfIndexedRandomDocuments();
@@ -1257,11 +1262,12 @@ public void testPeerRecoveryRetentionLeases() throws IOException {
      */
     public void testOperationBasedRecovery() throws Exception {
         if (isRunningAgainstOldCluster()) {
-            createIndex(index, Settings.builder()
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
-                .build());
+            Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1);
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+            }
+            createIndex(index, settings.build());
             ensureGreen(index);
             int committedDocs = randomIntBetween(100, 200);
             for (int i = 0; i < committedDocs; i++) {
@@ -1309,4 +1315,77 @@ public void testTurnOffTranslogRetentionAfterUpgraded() throws Exception {
             ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index);
         }
     }
+
+    public void testResize() throws Exception {
+        int numDocs;
+        if (isRunningAgainstOldCluster()) {
+            final Settings.Builder settings = Settings.builder()
+                .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3)
+                .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1);
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
+            }
+            createIndex(index, settings.build());
+            numDocs = randomIntBetween(10, 1000);
+            for (int i = 0; i < numDocs; i++) {
+                indexDocument(Integer.toString(i));
+                if (rarely()) {
+                    flush(index, randomBoolean());
+                }
+            }
+            saveInfoDocument("num_doc_" + index, Integer.toString(numDocs));
+            ensureGreen(index);
+        } else {
+            ensureGreen(index);
+            numDocs = Integer.parseInt(loadInfoDocument("num_doc_" + index));
+            int moreDocs = randomIntBetween(0, 100);
+            for (int i = 0; i < moreDocs; i++) {
+                indexDocument(Integer.toString(numDocs + i));
+                if (rarely()) {
+                    flush(index, randomBoolean());
+                }
+            }
+            Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
+            updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
+            client().performRequest(updateSettingsRequest);
+            {
+                final String target = index + "_shrunken";
+                Request shrinkRequest = new Request("PUT", "/" + index + "/_shrink/" + target);
+                Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1);
+                if (randomBoolean()) {
+                    settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+                }
+                shrinkRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
+                client().performRequest(shrinkRequest);
+                ensureGreenLongWait(target);
+                assertNumHits(target, numDocs + moreDocs, 1);
+            }
+            {
+                final String target = index + "_split";
+                Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 6);
+                if (randomBoolean()) {
+                    settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+                }
+                Request splitRequest = new Request("PUT", "/" + index + "/_split/" + target);
+                splitRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
+                client().performRequest(splitRequest);
+                ensureGreenLongWait(target);
+                assertNumHits(target, numDocs + moreDocs, 6);
+            }
+            {
+                final String target = index + "_cloned";
+                client().performRequest(new Request("PUT", "/" + index + "/_clone/" + target));
+                ensureGreenLongWait(target);
+                assertNumHits(target, numDocs + moreDocs, 3);
+            }
+        }
+    }
+
+    private void assertNumHits(String index, int numHits, int totalShards) throws IOException {
+        Map<String, Object> resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
+        assertNoFailures(resp);
+        assertThat(XContentMapValues.extractValue("_shards.total", resp), equalTo(totalShards));
+        assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards));
+        assertThat(extractTotalHits(resp), equalTo(numHits));
+    }
 }

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -293,7 +293,7 @@ public void testRecovery() throws Exception {
                 // before timing out
                 .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
                 .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
-            if (randomBoolean()) {
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
                 settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
             }
             createIndex(index, settings.build());
@@ -327,8 +327,10 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio
                 .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5))
                 .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion
                 .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
-                .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
-                .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+                .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+            }
             createIndex(index, settings.build());
             int numDocs = randomInt(10);
             indexDocs(index, 0, numDocs);
@@ -350,8 +352,10 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti
                     .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5))
                     .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1))
                     .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
-                    .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
-                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+                    .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
+                if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                    settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+                }
                 createIndex(index, settings.build());
                 int numDocs = randomInt(10);
                 indexDocs(index, 0, numDocs);
@@ -635,10 +639,13 @@ private void assertNoopRecoveries(String indexName, Predicate<String> targetNode
     public void testOperationBasedRecovery() throws Exception {
         final String index = "test_operation_based_recovery";
         if (CLUSTER_TYPE == ClusterType.OLD) {
-            createIndex(index, Settings.builder()
+            final Settings.Builder settings = Settings.builder()
                 .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
-                .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build());
+                .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2);
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+            }
+            createIndex(index, settings.build());
             ensureGreen(index);
             indexDocs(index, 0, randomIntBetween(100, 200));
             flush(index, randomBoolean());
@@ -714,7 +721,7 @@ public void testSoftDeletesDisabledWarning() throws Exception {
         if (CLUSTER_TYPE == ClusterType.OLD) {
             boolean softDeletesEnabled = true;
             Settings.Builder settings = Settings.builder();
-            if (randomBoolean()) {
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
                 softDeletesEnabled = randomBoolean();
                 settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled);
             }

rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml

@@ -123,14 +123,10 @@
 ---
 "Create index without soft deletes":
   - skip:
-      version:  " - 7.5.99"
-      reason:   "indices without soft deletes are deprecated in 7.6"
-      features: "warnings"
-
+      version:  " - 7.9.99"
+      reason:   "indices without soft-deletes is no longer supported "
   - do:
-      warnings:
-        - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
-          Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index].
+      catch: /illegal_argument_exception/
       indices.create:
         index: test_index
         body:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml

@@ -1,83 +1,5 @@
 ---
-"Translog retention without soft_deletes":
-  - skip:
-      version:  " - 7.5.99"
-      reason:   "indices without soft deletes are deprecated in 7.6"
-      features: "warnings"
-
-  - do:
-      indices.create:
-        index: test
-        body:
-          settings:
-            soft_deletes.enabled: false
-      warnings:
-        - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
-          Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
-  - do:
-      cluster.health:
-        wait_for_no_initializing_shards: true
-        wait_for_events: languid
-  - do:
-      indices.stats:
-        metric: [ translog ]
-  - set: { indices.test.primaries.translog.size_in_bytes: creation_size }
-
-  - do:
-      index:
-        index: test
-        id:    1
-        body:  { "foo": "bar" }
-
-  - do:
-      indices.stats:
-        metric: [ translog ]
-  - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size }
-  - match: { indices.test.primaries.translog.operations: 1 }
-# we can't check this yet as creation size will contain two empty translog generations. A single
-# non empty generation with one op may be smaller or larger than that.
-#  - gt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
-  - match: { indices.test.primaries.translog.uncommitted_operations: 1 }
-
-  - do:
-      indices.flush:
-        index: test
-
-  - do:
-      indices.stats:
-        metric: [ translog ]
-  - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size }
-  - match: { indices.test.primaries.translog.operations: 1 }
-  ## creation translog size has some overhead due to an initial empty generation that will be trimmed later
-  - lt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
-  - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
-
-  - do:
-      indices.put_settings:
-        index: test
-        body:
-          index.translog.retention.size: -1
-          index.translog.retention.age: -1
-
-  - do:
-      indices.flush:
-        index: test
-        force: true # force flush as we don't have pending ops
-
-  - do:
-      indices.stats:
-        metric: [ translog ]
-  ## creation translog size has some overhead due to an initial empty generation that will be trimmed later
-  - lte: { indices.test.primaries.translog.size_in_bytes: $creation_size }
-  - match: { indices.test.primaries.translog.operations: 0 }
-  - lte: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
-  - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
-
----
-"Translog retention with soft_deletes":
-  - skip:
-      version: " - 7.3.99"
-      reason:  "start ignoring translog retention policy with soft-deletes enabled in 7.4"
+"Translog retention":
   - do:
       indices.create:
         index: test
@@ -136,69 +58,7 @@
   - gte: { indices.test.primaries.translog.earliest_last_modified_age: 0 }
 
 ---
-"Translog stats on closed indices without soft-deletes":
-  - skip:
-      version: " - 7.5.99"
-      reason:  "indices without soft deletes are deprecated in 7.6"
-      features: "warnings"
-
-  - do:
-      indices.create:
-        index: test
-        body:
-          settings:
-            soft_deletes.enabled: false
-      warnings:
-        - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
-          Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
-
-  - do:
-      cluster.health:
-        wait_for_no_initializing_shards: true
-        wait_for_events: languid
-  - do:
-      index:
-        index: test
-        id:    1
-        body:  { "foo": "bar" }
-
-  - do:
-      index:
-        index: test
-        id:    2
-        body:  { "foo": "bar" }
-
-  - do:
-      index:
-        index: test
-        id:    3
-        body:  { "foo": "bar" }
-
-  - do:
-      indices.stats:
-        metric: [ translog ]
-  - match: { indices.test.primaries.translog.operations: 3 }
-  - match: { indices.test.primaries.translog.uncommitted_operations: 3 }
-
-  - do:
-      indices.close:
-        index: test
-        wait_for_active_shards: 1
-  - is_true: acknowledged
-
-  - do:
-      indices.stats:
-        metric: [ translog ]
-        expand_wildcards: all
-        forbid_closed_indices: false
-  - match: { indices.test.primaries.translog.operations: 3 }
-  - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
-
----
-"Translog stats on closed indices with soft-deletes":
-  - skip:
-      version: " - 7.3.99"
-      reason:  "start ignoring translog retention policy with soft-deletes enabled in 7.4"
+"Translog stats on closed indices":
   - do:
       indices.create:
         index: test

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -53,7 +53,6 @@
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -104,7 +103,6 @@
  */
 public class MetaDataCreateIndexService {
     private static final Logger logger = LogManager.getLogger(MetaDataCreateIndexService.class);
-    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger);
 
     public static final int MAX_INDEX_NAME_BYTES = 255;
 
@@ -439,10 +437,10 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
          * that will be used to create this index.
          */
         MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState);
-        if (indexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
-            DEPRECATION_LOGGER.deprecatedAndMaybeLog("soft_deletes_disabled",
-                "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " +
-                "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + request.index() + "].");
+        if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) == false
+            && IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(indexSettings).onOrAfter(Version.V_8_0_0)) {
+            throw new IllegalArgumentException("Creating indices with soft-deletes disabled is no longer supported. " +
+                "Please do not specify a value for setting [index.soft_deletes.enabled].");
         }
         return indexSettings;
     }