diff --git a/docs/reference/migration/migrate_8_0/indices.asciidoc b/docs/reference/migration/migrate_8_0/indices.asciidoc index ef5cd25f81808..03acac7753539 100644 --- a/docs/reference/migration/migrate_8_0/indices.asciidoc +++ b/docs/reference/migration/migrate_8_0/indices.asciidoc @@ -34,3 +34,12 @@ removed in 8.0. Synced flush was deprecated in 7.6 and is removed in 8.0. Use a regular flush instead as it has the same effect as a synced flush in 7.6 and later. + + +[float] +==== Indices with soft deletes disabled + +Creating indices with soft deletes disabled was deprecated in 7.6 and +is no longer supported in 8.0. The setting index.soft_deletes.enabled +can no longer be set to false. As the setting defaults to true, simply +leave the setting unset. diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 427383d7d8292..406099c82e09e 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -776,6 +776,11 @@ public void testSnapshotRestore() throws IOException { if (isRunningAgainstOldCluster()) { // Create the index count = between(200, 300); + Settings.Builder settings = Settings.builder(); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } + createIndex(index, settings.build()); indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject()); } else { count = countOfIndexedRandomDocuments(); @@ -1257,11 +1262,12 @@ public void testPeerRecoveryRetentionLeases() throws IOException { */ public void testOperationBasedRecovery() throws Exception { if (isRunningAgainstOldCluster()) { - createIndex(index, Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .build()); + Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } + createIndex(index, settings.build()); ensureGreen(index); int committedDocs = randomIntBetween(100, 200); for (int i = 0; i < committedDocs; i++) { @@ -1309,4 +1315,77 @@ public void testTurnOffTranslogRetentionAfterUpgraded() throws Exception { ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index); } } + + public void testResize() throws Exception { + int numDocs; + if (isRunningAgainstOldCluster()) { + final Settings.Builder settings = Settings.builder() + .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3) + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false); + } + createIndex(index, settings.build()); + numDocs = randomIntBetween(10, 1000); + for (int i = 0; i < numDocs; i++) { + indexDocument(Integer.toString(i)); + if (rarely()) { + flush(index, randomBoolean()); + } + } + saveInfoDocument("num_doc_" + index, Integer.toString(numDocs)); + ensureGreen(index); + } else { + ensureGreen(index); + numDocs = Integer.parseInt(loadInfoDocument("num_doc_" + index)); + int moreDocs = randomIntBetween(0, 100); + for (int i = 0; i < moreDocs; i++) { + indexDocument(Integer.toString(numDocs + i)); + if (rarely()) { + flush(index, randomBoolean()); + } + } + Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings"); + updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}"); + client().performRequest(updateSettingsRequest); + { + final String target = index + "_shrunken"; + Request shrinkRequest = new Request("PUT", "/" + index + "/_shrink/" + target); + Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1); + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); + } + shrinkRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}"); + client().performRequest(shrinkRequest); + ensureGreenLongWait(target); + assertNumHits(target, numDocs + moreDocs, 1); + } + { + final String target = index + "_split"; + Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 6); + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); + } + Request splitRequest = new Request("PUT", "/" + index + "/_split/" + target); + splitRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}"); + client().performRequest(splitRequest); + ensureGreenLongWait(target); + assertNumHits(target, numDocs + moreDocs, 6); + } + { + final String target = index + "_cloned"; + client().performRequest(new Request("PUT", "/" + index + "/_clone/" + target)); + ensureGreenLongWait(target); + assertNumHits(target, numDocs + moreDocs, 3); + } + } + } + + private void assertNumHits(String index, int numHits, int totalShards) throws IOException { + Map resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))); + assertNoFailures(resp); + assertThat(XContentMapValues.extractValue("_shards.total", resp), equalTo(totalShards)); + assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards)); + assertThat(extractTotalHits(resp), equalTo(numHits)); + } } diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index d0892fd914f57..dfca9dd74a635 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -293,7 +293,7 @@ public void testRecovery() throws Exception { // before timing out .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster - if (randomBoolean()) { + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); } createIndex(index, settings.build()); @@ -327,8 +327,10 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5)) .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") - .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } createIndex(index, settings.build()); int numDocs = randomInt(10); indexDocs(index, 0, numDocs); @@ -350,8 +352,10 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5)) .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1)) .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") - .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } createIndex(index, settings.build()); int numDocs = randomInt(10); indexDocs(index, 0, numDocs); @@ -635,10 +639,13 @@ private void assertNoopRecoveries(String indexName, Predicate targetNode public void testOperationBasedRecovery() throws Exception { final String index = "test_operation_based_recovery"; if (CLUSTER_TYPE == ClusterType.OLD) { - createIndex(index, Settings.builder() + final Settings.Builder settings = Settings.builder() .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build()); + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } + createIndex(index, settings.build()); ensureGreen(index); indexDocs(index, 0, randomIntBetween(100, 200)); flush(index, randomBoolean()); @@ -714,7 +721,7 @@ public void testSoftDeletesDisabledWarning() throws Exception { if (CLUSTER_TYPE == ClusterType.OLD) { boolean softDeletesEnabled = true; Settings.Builder settings = Settings.builder(); - if (randomBoolean()) { + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { softDeletesEnabled = randomBoolean(); settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml index b5ce95c31a581..44ec99ce9f95d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -123,14 +123,10 @@ --- "Create index without soft deletes": - skip: - version: " - 7.5.99" - reason: "indices without soft deletes are deprecated in 7.6" - features: "warnings" - + version: " - 7.9.99" + reason: "indices without soft-deletes is no longer supported " - do: - warnings: - - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. - Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index]. + catch: /illegal_argument_exception/ indices.create: index: test_index body: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml index 0cc21eda1bdcd..fbbc49bbe59eb 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml @@ -1,83 +1,5 @@ --- -"Translog retention without soft_deletes": - - skip: - version: " - 7.5.99" - reason: "indices without soft deletes are deprecated in 7.6" - features: "warnings" - - - do: - indices.create: - index: test - body: - settings: - soft_deletes.enabled: false - warnings: - - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. - Please do not specify value for setting [index.soft_deletes.enabled] of index [test]. - - do: - cluster.health: - wait_for_no_initializing_shards: true - wait_for_events: languid - - do: - indices.stats: - metric: [ translog ] - - set: { indices.test.primaries.translog.size_in_bytes: creation_size } - - - do: - index: - index: test - id: 1 - body: { "foo": "bar" } - - - do: - indices.stats: - metric: [ translog ] - - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.operations: 1 } -# we can't check this yet as creation size will contain two empty translog generations. A single -# non empty generation with one op may be smaller or larger than that. -# - gt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.uncommitted_operations: 1 } - - - do: - indices.flush: - index: test - - - do: - indices.stats: - metric: [ translog ] - - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.operations: 1 } - ## creation translog size has some overhead due to an initial empty generation that will be trimmed later - - lt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.uncommitted_operations: 0 } - - - do: - indices.put_settings: - index: test - body: - index.translog.retention.size: -1 - index.translog.retention.age: -1 - - - do: - indices.flush: - index: test - force: true # force flush as we don't have pending ops - - - do: - indices.stats: - metric: [ translog ] - ## creation translog size has some overhead due to an initial empty generation that will be trimmed later - - lte: { indices.test.primaries.translog.size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.operations: 0 } - - lte: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.uncommitted_operations: 0 } - ---- -"Translog retention with soft_deletes": - - skip: - version: " - 7.3.99" - reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4" +"Translog retention": - do: indices.create: index: test @@ -136,69 +58,7 @@ - gte: { indices.test.primaries.translog.earliest_last_modified_age: 0 } --- -"Translog stats on closed indices without soft-deletes": - - skip: - version: " - 7.5.99" - reason: "indices without soft deletes are deprecated in 7.6" - features: "warnings" - - - do: - indices.create: - index: test - body: - settings: - soft_deletes.enabled: false - warnings: - - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. - Please do not specify value for setting [index.soft_deletes.enabled] of index [test]. - - - do: - cluster.health: - wait_for_no_initializing_shards: true - wait_for_events: languid - - do: - index: - index: test - id: 1 - body: { "foo": "bar" } - - - do: - index: - index: test - id: 2 - body: { "foo": "bar" } - - - do: - index: - index: test - id: 3 - body: { "foo": "bar" } - - - do: - indices.stats: - metric: [ translog ] - - match: { indices.test.primaries.translog.operations: 3 } - - match: { indices.test.primaries.translog.uncommitted_operations: 3 } - - - do: - indices.close: - index: test - wait_for_active_shards: 1 - - is_true: acknowledged - - - do: - indices.stats: - metric: [ translog ] - expand_wildcards: all - forbid_closed_indices: false - - match: { indices.test.primaries.translog.operations: 3 } - - match: { indices.test.primaries.translog.uncommitted_operations: 0 } - ---- -"Translog stats on closed indices with soft-deletes": - - skip: - version: " - 7.3.99" - reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4" +"Translog stats on closed indices": - do: indices.create: index: test diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index ded77ce4287d6..f605ee9de75eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -53,7 +53,6 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -104,7 +103,6 @@ */ public class MetaDataCreateIndexService { private static final Logger logger = LogManager.getLogger(MetaDataCreateIndexService.class); - private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); public static final int MAX_INDEX_NAME_BYTES = 255; @@ -437,10 +435,10 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu * that will be used to create this index. */ MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState); - if (indexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) { - DEPRECATION_LOGGER.deprecatedAndMaybeLog("soft_deletes_disabled", - "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " + - "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + request.index() + "]."); + if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) == false + && IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(indexSettings).onOrAfter(Version.V_8_0_0)) { + throw new IllegalArgumentException("Creating indices with soft-deletes disabled is no longer supported. " + + "Please do not specify a value for setting [index.soft_deletes.enabled]."); } return indexSettings; } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index cce17b8f441e7..0ffbbedeea32e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -227,11 +227,10 @@ public final class IndexSettings { /** * Specifies if the index should use soft-delete instead of hard-delete for update/delete operations. - * Soft-deletes is enabled by default for 7.0+ indices. + * Soft-deletes is enabled by default for 7.0 indices and mandatory for 8.0 indices. */ - public static final Setting INDEX_SOFT_DELETES_SETTING = Setting.boolSetting("index.soft_deletes.enabled", - settings -> Boolean.toString(IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.V_7_0_0)), - Property.IndexScope, Property.Final); + public static final Setting INDEX_SOFT_DELETES_SETTING = + Setting.boolSetting("index.soft_deletes.enabled", true, Property.IndexScope, Property.Final); /** * Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted @@ -492,6 +491,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti mergeSchedulerConfig = new MergeSchedulerConfig(this); gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis(); softDeleteEnabled = scopedSettings.get(INDEX_SOFT_DELETES_SETTING); + assert softDeleteEnabled || version.before(Version.V_8_0_0) : "soft deletes must be enabled in version " + version; softDeleteRetentionOperations = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING); retentionLeaseMillis = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING).millis(); warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index e90f710af0f45..416491ce31ba5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -779,14 +779,12 @@ public void testBuildIndexMetadata() { Settings indexSettings = Settings.builder() .put("index.version.created", Version.CURRENT) - .put(INDEX_SOFT_DELETES_SETTING.getKey(), false) .put(SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_NUMBER_OF_SHARDS, 1) .build(); List aliases = List.of(AliasMetaData.builder("alias1").build()); IndexMetaData indexMetaData = buildIndexMetaData("test", aliases, () -> null, indexSettings, 4, sourceIndexMetaData); - assertThat(indexMetaData.getSettings().getAsBoolean(INDEX_SOFT_DELETES_SETTING.getKey(), true), is(false)); assertThat(indexMetaData.getAliases().size(), is(1)); assertThat(indexMetaData.getAliases().keys().iterator().next().value, is("alias1")); assertThat("The source index primary term must be used", indexMetaData.primaryTerm(0), is(3L)); @@ -828,19 +826,15 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() { assertThat(targetRoutingNumberOfShards, is(6)); } - public void testSoftDeletesDisabledDeprecation() { - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); - assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " - + "Please do not specify value for setting [index.soft_deletes.enabled] of index [test]."); - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - if (randomBoolean()) { - request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build()); - } - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + public void testRejectWithSoftDeletesDisabled() { + final IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + }); + assertThat(error.getMessage(), equalTo("Creating indices with soft-deletes disabled is no longer supported. " + + "Please do not specify a value for setting [index.soft_deletes.enabled].")); } private IndexTemplateMetaData addMatchingTemplate(Consumer configurator) { diff --git a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java index cf0c766972e09..67c9c3a7844bb 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java @@ -77,7 +77,6 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { assertAcked( client().admin().indices().prepareCreate(indexName) .setSettings(Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) .put(IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.getKey(), 1.0f) @@ -206,7 +205,6 @@ public void testFullClusterRestartPerformNoopRecovery() throws Exception { assertAcked( client().admin().indices().prepareCreate(indexName) .setSettings(Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb") .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) @@ -243,7 +241,6 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception { assertAcked( client().admin().indices().prepareCreate(indexName) .setSettings(Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb") .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) @@ -324,7 +321,6 @@ public void testPeerRecoveryForClosedIndices() throws Exception { createIndex(indexName, Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms") .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms") .build()); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 1a19647ea7f78..57dfebbd231c0 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.translog.Translog; @@ -572,38 +571,4 @@ public void testIgnoreTranslogRetentionSettingsIfSoftDeletesEnabled() { assertThat(indexSettings.getTranslogRetentionAge().millis(), equalTo(-1L)); assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L)); } - - public void testUpdateTranslogRetentionSettingsWithSoftDeletesDisabled() { - Settings.Builder settings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); - - TimeValue ageSetting = TimeValue.timeValueHours(12); - if (randomBoolean()) { - ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000)); - settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting); - } - ByteSizeValue sizeSetting = new ByteSizeValue(512, ByteSizeUnit.MB); - if (randomBoolean()) { - sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024)); - settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting); - } - IndexMetaData metaData = newIndexMeta("index", settings.build()); - IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY); - assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); - assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); - - Settings.Builder newSettings = Settings.builder().put(settings.build()); - if (randomBoolean()) { - ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000)); - newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting); - } - if (randomBoolean()) { - sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024)); - newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting); - } - indexSettings.updateIndexMetaData(newIndexMeta("index", newSettings.build())); - assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); - assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); - } } diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 2d5dc675f025b..bcf20e1fd1f0b 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -5477,8 +5477,9 @@ public long softUpdateDocuments(Term term, Iterable operations = generateHistoryOnReplica(between(1, 500), randomBoolean(), randomBoolean(), randomBoolean()); final IndexMetaData indexMetaData = IndexMetaData.builder(defaultSettings.getIndexMetaData()) - .settings(Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)) - .build(); + .settings(Settings.builder().put(defaultSettings.getSettings()) + .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0)) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)).build(); final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData); try (Store store = createStore()) { EngineConfig config = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get); diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java index 53a96e531585a..ec10e2a5e8444 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.DocIdSeqNoAndSource; @@ -108,68 +107,6 @@ public void testIndexingDuringFileRecovery() throws Exception { } } - public void testRecoveryOfDisconnectedReplica() throws Exception { - Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build(); - try (ReplicationGroup shards = createGroup(1, settings)) { - shards.startAll(); - int docs = shards.indexDocs(randomInt(50)); - shards.flush(); - final IndexShard originalReplica = shards.getReplicas().get(0); - for (int i = 0; i < randomInt(2); i++) { - final int indexedDocs = shards.indexDocs(randomInt(5)); - docs += indexedDocs; - - final boolean flush = randomBoolean(); - if (flush) { - originalReplica.flush(new FlushRequest()); - } - } - - // simulate a background global checkpoint sync at which point we expect the global checkpoint to advance on the replicas - shards.syncGlobalCheckpoint(); - long globalCheckpointOnReplica = originalReplica.getLastSyncedGlobalCheckpoint(); - Optional safeCommitOnReplica = - originalReplica.store().findSafeIndexCommit(globalCheckpointOnReplica); - assertTrue(safeCommitOnReplica.isPresent()); - shards.removeReplica(originalReplica); - - final int missingOnReplica = shards.indexDocs(randomInt(5)); - docs += missingOnReplica; - - final boolean translogTrimmed; - if (randomBoolean()) { - shards.flush(); - translogTrimmed = randomBoolean(); - if (translogTrimmed) { - final Translog translog = getTranslog(shards.getPrimary()); - translog.getDeletionPolicy().setRetentionAgeInMillis(0); - translog.trimUnreferencedReaders(); - } - } else { - translogTrimmed = false; - } - originalReplica.close("disconnected", false); - IOUtils.close(originalReplica.store()); - final IndexShard recoveredReplica = - shards.addReplicaWithExistingPath(originalReplica.shardPath(), originalReplica.routingEntry().currentNodeId()); - shards.recoverReplica(recoveredReplica); - if (translogTrimmed && missingOnReplica > 0) { - // replica has something to catch up with, but since we trimmed the primary translog, we should fall back to full recovery - assertThat(recoveredReplica.recoveryState().getIndex().fileDetails(), not(empty())); - } else { - assertThat(recoveredReplica.recoveryState().getIndex().fileDetails(), empty()); - assertThat(recoveredReplica.recoveryState().getTranslog().recoveredOperations(), - equalTo(Math.toIntExact(docs - 1 - safeCommitOnReplica.get().localCheckpoint))); - assertThat(recoveredReplica.recoveryState().getTranslog().totalLocal(), - equalTo(Math.toIntExact(globalCheckpointOnReplica - safeCommitOnReplica.get().localCheckpoint))); - } - - docs += shards.indexDocs(randomInt(5)); - - shards.assertAllEqual(docs); - } - } - /* * Simulate a scenario with two replicas where one of the replicas receives an extra document, the other replica is promoted on primary * failure, the receiving replica misses the primary/replica re-sync and then recovers from the primary. We expect that a diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 86fa8c613b628..4f90b042ebd10 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -844,7 +844,6 @@ public void testLimitNumberOfRetainedTranslogFiles() throws Exception { Settings.Builder settings = Settings.builder() .put(SETTING_NUMBER_OF_SHARDS, 1) .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) .put(IndexSettings.INDEX_TRANSLOG_RETENTION_TOTAL_FILES_SETTING.getKey(), translogRetentionTotalFiles); if (randomBoolean()) { settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), new ByteSizeValue(between(1, 1024 * 1024))); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java index 31bdfce261ad9..28995efc4be20 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java @@ -303,21 +303,6 @@ public void testRetentionLeaseStats() throws IOException { } } - public void testRetentionLeasesActionsFailWithSoftDeletesDisabled() throws Exception { - IndexShard shard = newStartedShard(true, Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); - assertThat(expectThrows(AssertionError.class, () -> shard.addRetentionLease(randomAlphaOfLength(10), - randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE), "test", ActionListener.wrap(() -> {}))).getMessage(), - equalTo("retention leases requires soft deletes but [index] does not have soft deletes enabled")); - assertThat(expectThrows(AssertionError.class, () -> shard.renewRetentionLease( - randomAlphaOfLength(10), randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE), "test")).getMessage(), - equalTo("retention leases requires soft deletes but [index] does not have soft deletes enabled")); - assertThat(expectThrows(AssertionError.class, () -> shard.removeRetentionLease( - randomAlphaOfLength(10), ActionListener.wrap(() -> {}))).getMessage(), - equalTo("retention leases requires soft deletes but [index] does not have soft deletes enabled")); - shard.syncRetentionLeases(); - closeShards(shard); - } - private void assertRetentionLeases( final IndexShard indexShard, final int size, diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 6a8a63ce17f32..39f120b2fa3a3 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2113,8 +2113,7 @@ public void testPrimaryHandOffUpdatesLocalCheckpoint() throws IOException { } IndexShardTestCase.updateRoutingEntry(primarySource, primarySource.routingEntry().relocate(randomAlphaOfLength(10), -1)); - final IndexShard primaryTarget = newShard(primarySource.routingEntry().getTargetRelocatingShard(), Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), primarySource.indexSettings().isSoftDeleteEnabled()).build()); + final IndexShard primaryTarget = newShard(primarySource.routingEntry().getTargetRelocatingShard()); updateMappings(primaryTarget, primarySource.indexSettings().getIndexMetaData()); recoverReplica(primaryTarget, primarySource, true); @@ -2128,13 +2127,11 @@ public void testPrimaryHandOffUpdatesLocalCheckpoint() throws IOException { /* This test just verifies that we fill up local checkpoint up to max seen seqID on primary recovery */ public void testRecoverFromStoreWithNoOps() throws IOException { - final Settings settings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build(); - final IndexShard shard = newStartedShard(true, settings); + final IndexShard shard = newStartedShard(true); indexDoc(shard, "_doc", "0"); indexDoc(shard, "_doc", "1"); // start a replica shard and index the second doc - final IndexShard otherShard = newStartedShard(false, settings); + final IndexShard otherShard = newStartedShard(false); updateMappings(otherShard, shard.indexSettings().getIndexMetaData()); SourceToParse sourceToParse = new SourceToParse(shard.shardId().getIndexName(), "1", new BytesArray("{}"), XContentType.JSON); @@ -2314,8 +2311,7 @@ public void testRecoveryFailsAfterMovingToRelocatedState() throws InterruptedExc public void testRestoreShard() throws IOException { final IndexShard source = newStartedShard(true); - IndexShard target = newStartedShard(true, Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), source.indexSettings().isSoftDeleteEnabled()).build()); + IndexShard target = newStartedShard(true); indexDoc(source, "_doc", "0"); EngineTestCase.generateNewSeqNo(source.getEngine()); // create a gap in the history diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 999028cb083fd..8ddb59ef5db1a 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -446,7 +446,6 @@ public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOE final IndexMetaData.Builder indexMetaData = IndexMetaData.builder("test").settings(Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, between(0,5)) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1,5)) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random())) .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))); if (randomBoolean()) { diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java index 05aebcc459a6e..5cadc31d7946e 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java @@ -28,6 +28,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.bulk.BulkShardRequest; @@ -55,6 +56,7 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.SnapshotMatchers; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.HashMap; @@ -132,7 +134,9 @@ public void testRecoveryWithOutOfOrderDeleteWithTranslog() throws Exception { * - index #5 * - If flush and the translog retention disabled, delete #1 will be removed while index #0 is still retained and replayed. */ - Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build(); + Settings settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0)) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build(); try (ReplicationGroup shards = createGroup(1, settings)) { shards.startAll(); // create out of order delete and index op on replica diff --git a/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java index c95b61f7e6a90..df908fd4f8cce 100644 --- a/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java +++ b/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -354,8 +353,6 @@ public void testTranslogStats() throws Exception { createIndex(indexName, Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .build()); - boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get( - client().admin().indices().prepareGetSettings(indexName).get().getIndexToSettings().get(indexName)); final int nbDocs = randomIntBetween(0, 50); int uncommittedOps = 0; @@ -375,8 +372,8 @@ public void testTranslogStats() throws Exception { assertBusy(() -> { IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get(); assertThat(stats.getIndex(indexName), notNullValue()); - assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo( - softDeletesEnabled ? uncommittedTranslogOps : nbDocs)); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), + equalTo(uncommittedTranslogOps)); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedTranslogOps)); }); @@ -386,8 +383,7 @@ public void testTranslogStats() throws Exception { IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).setIndicesOptions(indicesOptions) .clear().setTranslog(true).get(); assertThat(stats.getIndex(indexName), notNullValue()); - assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), - equalTo(softDeletesEnabled ? 0 : nbDocs)); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(0)); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0)); } } diff --git a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 7a3de4f4c977e..2a29582fdefd3 100644 --- a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -47,7 +47,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLease; import org.elasticsearch.index.shard.IndexEventListener; @@ -627,7 +626,6 @@ public void testRelocationEstablishedPeerRecoveryRetentionLeases() throws Except ensureStableCluster(halfNodes * 2); assertAcked( client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, halfNodes - 1)) .put("index.routing.allocation.include.color", "blue"))); ensureGreen("test"); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 6ec344783c8f2..f2a44b766e73e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -184,9 +184,7 @@ protected Settings indexSettings() { .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(), between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY))) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000)) + .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)) .build(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index c067c0717dcce..54bc1c3dfcda2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -133,9 +133,7 @@ protected IndexMetaData buildIndexMetaData(int replicas, Settings indexSettings, Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000)) + .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)) .put(indexSettings) .build(); IndexMetaData.Builder metaData = IndexMetaData.builder(index.getName()) diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index d0c5bf1ca53a1..c215078635915 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -249,7 +249,6 @@ protected IndexShard newShard( Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000)) .put(settings) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 038efa4d4159f..dd4f937039afc 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -670,7 +670,6 @@ public Settings indexSettings() { } // always default delayed allocation to 0 to make sure we have tests are not delayed builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0); - builder.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); if (randomBoolean()) { builder.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 6265dbe559fa5..79ded94250da6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -95,10 +95,8 @@ protected void startNode(long seed) throws Exception { .preparePutTemplate("random-soft-deletes-template") .setPatterns(Collections.singletonList("*")) .setOrder(0) - .setSettings(Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000)) - ).get(); + .setSettings(Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000))) + .get(); } private static void stopNode() throws IOException, InterruptedException { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index f944dd70813bc..dcbcad3b2048d 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -358,41 +358,6 @@ public void testConflictingPatterns() throws Exception { assertFalse(ESIntegTestCase.indexExists("copy-logs-201801", followerClient())); } - public void testAutoFollowSoftDeletesDisabled() throws Exception { - putAutoFollowPatterns("my-pattern1", new String[] {"logs-*"}); - - // Soft deletes are disabled: - Settings leaderIndexSettings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) - .build(); - createLeaderIndex("logs-20200101", leaderIndexSettings); - assertBusy(() -> { - AutoFollowStats autoFollowStats = getAutoFollowStats(); - assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(0L)); - assertThat(autoFollowStats.getNumberOfFailedFollowIndices(), equalTo(1L)); - assertThat(autoFollowStats.getRecentAutoFollowErrors().size(), equalTo(1)); - ElasticsearchException failure = autoFollowStats.getRecentAutoFollowErrors().firstEntry().getValue().v2(); - assertThat(failure.getMessage(), equalTo("index [logs-20200101] cannot be followed, " + - "because soft deletes are not enabled")); - assertFalse(ESIntegTestCase.indexExists("copy-logs-20200101", followerClient())); - }); - - // Soft deletes are enabled: - leaderIndexSettings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) - .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) - .build(); - createLeaderIndex("logs-20200102", leaderIndexSettings); - assertBusy(() -> { - AutoFollowStats autoFollowStats = getAutoFollowStats(); - assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(1L)); - assertTrue(ESIntegTestCase.indexExists("copy-logs-20200102", followerClient())); - }); - } - public void testPauseAndResumeAutoFollowPattern() throws Exception { final Settings leaderIndexSettings = Settings.builder() .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java index 3a52950d1788b..b623be7ccbe0c 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.CcrSingleNodeTestCase; import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction; @@ -78,21 +77,6 @@ public void testFollowIndex() throws Exception { ensureEmptyWriteBuffers(); } - public void testDoNotCreateFollowerIfLeaderDoesNotHaveSoftDeletes() throws Exception { - final String leaderIndexSettings = getIndexSettings(2, 0, - singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "false")); - assertAcked(client().admin().indices().prepareCreate("leader-index").setSource(leaderIndexSettings, XContentType.JSON)); - ResumeFollowAction.Request followRequest = getResumeFollowRequest("follower"); - followRequest.setFollowerIndex("follower-index"); - PutFollowAction.Request putFollowRequest = getPutFollowRequest("leader", "follower"); - putFollowRequest.setLeaderIndex("leader-index"); - putFollowRequest.setFollowerIndex("follower-index"); - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, - () -> client().execute(PutFollowAction.INSTANCE, putFollowRequest).actionGet()); - assertThat(error.getMessage(), equalTo("leader index [leader-index] does not have soft deletes enabled")); - assertThat(ESIntegTestCase.indexExists("follower-index", client()), equalTo(false)); - } - public void testRemoveRemoteConnection() throws Exception { PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); request.setName("my_pattern");