From 42033d12c0da33401b5132d88d741d64a5496f57 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Jan 2020 12:20:02 -0500 Subject: [PATCH 1/8] Make soft-deletes mandatory in 8.0 --- .../migration/migrate_8_0/indices.asciidoc | 8 + .../upgrades/FullClusterRestartIT.java | 11 +- .../elasticsearch/upgrades/RecoveryIT.java | 25 +-- .../test/indices.create/10_basic.yml | 10 +- .../test/indices.stats/20_translog.yml | 144 +----------------- .../indices/shrink/TransportResizeAction.java | 4 +- .../metadata/MetaDataCreateIndexService.java | 9 +- .../elasticsearch/index/IndexSettings.java | 8 +- .../snapshots/RestoreService.java | 4 +- .../MetaDataCreateIndexServiceTests.java | 24 ++- .../gateway/ReplicaShardAllocatorIT.java | 4 - .../index/IndexSettingsTests.java | 35 ----- .../index/engine/InternalEngineTests.java | 5 +- .../RecoveryDuringReplicationTests.java | 3 +- .../index/shard/IndexShardIT.java | 1 - .../shard/IndexShardRetentionLeaseTests.java | 15 -- .../index/shard/IndexShardTests.java | 12 +- .../recovery/RecoverySourceHandlerTests.java | 1 - .../indices/recovery/RecoveryTests.java | 6 +- .../indices/state/OpenCloseIndexIT.java | 10 +- .../elasticsearch/recovery/RelocationIT.java | 2 - .../index/engine/EngineTestCase.java | 4 +- .../ESIndexLevelReplicationTestCase.java | 4 +- .../index/shard/IndexShardTestCase.java | 1 - .../elasticsearch/test/ESIntegTestCase.java | 1 - .../test/ESSingleNodeTestCase.java | 6 +- .../elasticsearch/xpack/ccr/AutoFollowIT.java | 35 ----- .../xpack/ccr/LocalIndexFollowingIT.java | 16 -- 28 files changed, 73 insertions(+), 335 deletions(-) diff --git a/docs/reference/migration/migrate_8_0/indices.asciidoc b/docs/reference/migration/migrate_8_0/indices.asciidoc index ef5cd25f81808..64bc11f9c79fa 100644 --- a/docs/reference/migration/migrate_8_0/indices.asciidoc +++ b/docs/reference/migration/migrate_8_0/indices.asciidoc @@ -34,3 +34,11 @@ removed in 8.0. Synced flush was deprecated in 7.6 and is removed in 8.0. Use a regular flush instead as it has the same effect as a synced flush in 7.6 and later. + + +[float] +==== Indices with soft deletes disabled + +Creating indices with soft deletes disabled was deprecated in 7.6 and +is no longer supported in 8.0. Do not specify value for the setting +`[index.soft_deletes.enabled]`. diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 427383d7d8292..4e609943f0ba8 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -1257,11 +1257,12 @@ public void testPeerRecoveryRetentionLeases() throws IOException { */ public void testOperationBasedRecovery() throws Exception { if (isRunningAgainstOldCluster()) { - createIndex(index, Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .build()); + Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } + createIndex(index, settings.build()); ensureGreen(index); int committedDocs = randomIntBetween(100, 200); for (int i = 0; i < committedDocs; i++) { diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index d0892fd914f57..dfca9dd74a635 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -293,7 +293,7 @@ public void testRecovery() throws Exception { // before timing out .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster - if (randomBoolean()) { + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); } createIndex(index, settings.build()); @@ -327,8 +327,10 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5)) .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") - .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } createIndex(index, settings.build()); int numDocs = randomInt(10); indexDocs(index, 0, numDocs); @@ -350,8 +352,10 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5)) .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1)) .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") - .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } createIndex(index, settings.build()); int numDocs = randomInt(10); indexDocs(index, 0, numDocs); @@ -635,10 +639,13 @@ private void assertNoopRecoveries(String indexName, Predicate targetNode public void testOperationBasedRecovery() throws Exception { final String index = "test_operation_based_recovery"; if (CLUSTER_TYPE == ClusterType.OLD) { - createIndex(index, Settings.builder() + final Settings.Builder settings = Settings.builder() .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build()); + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } + createIndex(index, settings.build()); ensureGreen(index); indexDocs(index, 0, randomIntBetween(100, 200)); flush(index, randomBoolean()); @@ -714,7 +721,7 @@ public void testSoftDeletesDisabledWarning() throws Exception { if (CLUSTER_TYPE == ClusterType.OLD) { boolean softDeletesEnabled = true; Settings.Builder settings = Settings.builder(); - if (randomBoolean()) { + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { softDeletesEnabled = randomBoolean(); settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml index b5ce95c31a581..44ec99ce9f95d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -123,14 +123,10 @@ --- "Create index without soft deletes": - skip: - version: " - 7.5.99" - reason: "indices without soft deletes are deprecated in 7.6" - features: "warnings" - + version: " - 7.9.99" + reason: "indices without soft-deletes is no longer supported " - do: - warnings: - - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. - Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index]. + catch: /illegal_argument_exception/ indices.create: index: test_index body: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml index 0cc21eda1bdcd..fbbc49bbe59eb 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml @@ -1,83 +1,5 @@ --- -"Translog retention without soft_deletes": - - skip: - version: " - 7.5.99" - reason: "indices without soft deletes are deprecated in 7.6" - features: "warnings" - - - do: - indices.create: - index: test - body: - settings: - soft_deletes.enabled: false - warnings: - - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. - Please do not specify value for setting [index.soft_deletes.enabled] of index [test]. - - do: - cluster.health: - wait_for_no_initializing_shards: true - wait_for_events: languid - - do: - indices.stats: - metric: [ translog ] - - set: { indices.test.primaries.translog.size_in_bytes: creation_size } - - - do: - index: - index: test - id: 1 - body: { "foo": "bar" } - - - do: - indices.stats: - metric: [ translog ] - - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.operations: 1 } -# we can't check this yet as creation size will contain two empty translog generations. A single -# non empty generation with one op may be smaller or larger than that. -# - gt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.uncommitted_operations: 1 } - - - do: - indices.flush: - index: test - - - do: - indices.stats: - metric: [ translog ] - - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.operations: 1 } - ## creation translog size has some overhead due to an initial empty generation that will be trimmed later - - lt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.uncommitted_operations: 0 } - - - do: - indices.put_settings: - index: test - body: - index.translog.retention.size: -1 - index.translog.retention.age: -1 - - - do: - indices.flush: - index: test - force: true # force flush as we don't have pending ops - - - do: - indices.stats: - metric: [ translog ] - ## creation translog size has some overhead due to an initial empty generation that will be trimmed later - - lte: { indices.test.primaries.translog.size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.operations: 0 } - - lte: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.uncommitted_operations: 0 } - ---- -"Translog retention with soft_deletes": - - skip: - version: " - 7.3.99" - reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4" +"Translog retention": - do: indices.create: index: test @@ -136,69 +58,7 @@ - gte: { indices.test.primaries.translog.earliest_last_modified_age: 0 } --- -"Translog stats on closed indices without soft-deletes": - - skip: - version: " - 7.5.99" - reason: "indices without soft deletes are deprecated in 7.6" - features: "warnings" - - - do: - indices.create: - index: test - body: - settings: - soft_deletes.enabled: false - warnings: - - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. - Please do not specify value for setting [index.soft_deletes.enabled] of index [test]. - - - do: - cluster.health: - wait_for_no_initializing_shards: true - wait_for_events: languid - - do: - index: - index: test - id: 1 - body: { "foo": "bar" } - - - do: - index: - index: test - id: 2 - body: { "foo": "bar" } - - - do: - index: - index: test - id: 3 - body: { "foo": "bar" } - - - do: - indices.stats: - metric: [ translog ] - - match: { indices.test.primaries.translog.operations: 3 } - - match: { indices.test.primaries.translog.uncommitted_operations: 3 } - - - do: - indices.close: - index: test - wait_for_active_shards: 1 - - is_true: acknowledged - - - do: - indices.stats: - metric: [ translog ] - expand_wildcards: all - forbid_closed_indices: false - - match: { indices.test.primaries.translog.operations: 3 } - - match: { indices.test.primaries.translog.uncommitted_operations: 0 } - ---- -"Translog stats on closed indices with soft-deletes": - - skip: - version: " - 7.3.99" - reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4" +"Translog stats on closed indices": - do: indices.create: index: test diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index f13c1096d5548..2d1a3964819c8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -175,9 +175,7 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize"); } } - if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(metaData.getSettings()) && - IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(targetIndexSettings) && - IndexSettings.INDEX_SOFT_DELETES_SETTING.get(targetIndexSettings) == false) { + if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(targetIndexSettings) == false) { throw new IllegalArgumentException("Can't disable [index.soft_deletes.enabled] setting on resize"); } String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index"; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index ded77ce4287d6..662f69763f2f2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -53,7 +53,6 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -104,7 +103,6 @@ */ public class MetaDataCreateIndexService { private static final Logger logger = LogManager.getLogger(MetaDataCreateIndexService.class); - private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); public static final int MAX_INDEX_NAME_BYTES = 255; @@ -437,10 +435,9 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu * that will be used to create this index. */ MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState); - if (indexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) { - DEPRECATION_LOGGER.deprecatedAndMaybeLog("soft_deletes_disabled", - "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " + - "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + request.index() + "]."); + if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) == false) { + throw new IllegalArgumentException("Creating indices with soft-deletes disabled is no longer supported. " + + "Please do not specify value for setting [index.soft_deletes.enabled]."); } return indexSettings; } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index cce17b8f441e7..0ffbbedeea32e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -227,11 +227,10 @@ public final class IndexSettings { /** * Specifies if the index should use soft-delete instead of hard-delete for update/delete operations. - * Soft-deletes is enabled by default for 7.0+ indices. + * Soft-deletes is enabled by default for 7.0 indices and mandatory for 8.0 indices. */ - public static final Setting INDEX_SOFT_DELETES_SETTING = Setting.boolSetting("index.soft_deletes.enabled", - settings -> Boolean.toString(IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.V_7_0_0)), - Property.IndexScope, Property.Final); + public static final Setting INDEX_SOFT_DELETES_SETTING = + Setting.boolSetting("index.soft_deletes.enabled", true, Property.IndexScope, Property.Final); /** * Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted @@ -492,6 +491,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti mergeSchedulerConfig = new MergeSchedulerConfig(this); gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis(); softDeleteEnabled = scopedSettings.get(INDEX_SOFT_DELETES_SETTING); + assert softDeleteEnabled || version.before(Version.V_8_0_0) : "soft deletes must be enabled in version " + version; softDeleteRetentionOperations = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING); retentionLeaseMillis = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING).millis(); warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 84c08e02bf179..3a14789a68152 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -65,7 +65,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.repositories.IndexId; @@ -124,8 +123,7 @@ public class RestoreService implements ClusterStateApplier { SETTING_NUMBER_OF_SHARDS, SETTING_VERSION_CREATED, SETTING_INDEX_UUID, - SETTING_CREATION_DATE, - IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey())); + SETTING_CREATION_DATE)); // It's OK to change some settings, but we shouldn't allow simply removing them private static final Set UNREMOVABLE_SETTINGS; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index e90f710af0f45..07f53e3025e10 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -779,14 +779,12 @@ public void testBuildIndexMetadata() { Settings indexSettings = Settings.builder() .put("index.version.created", Version.CURRENT) - .put(INDEX_SOFT_DELETES_SETTING.getKey(), false) .put(SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_NUMBER_OF_SHARDS, 1) .build(); List aliases = List.of(AliasMetaData.builder("alias1").build()); IndexMetaData indexMetaData = buildIndexMetaData("test", aliases, () -> null, indexSettings, 4, sourceIndexMetaData); - assertThat(indexMetaData.getSettings().getAsBoolean(INDEX_SOFT_DELETES_SETTING.getKey(), true), is(false)); assertThat(indexMetaData.getAliases().size(), is(1)); assertThat(indexMetaData.getAliases().keys().iterator().next().value, is("alias1")); assertThat("The source index primary term must be used", indexMetaData.primaryTerm(0), is(3L)); @@ -828,19 +826,15 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() { assertThat(targetRoutingNumberOfShards, is(6)); } - public void testSoftDeletesDisabledDeprecation() { - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); - assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " - + "Please do not specify value for setting [index.soft_deletes.enabled] of index [test]."); - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - if (randomBoolean()) { - request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build()); - } - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + public void testRejectWithSoftDeletesDisabled() { + final IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + }); + assertThat(error.getMessage(), equalTo("Creating indices with soft-deletes disabled is no longer supported. " + + "Please do not specify value for setting [index.soft_deletes.enabled].")); } private IndexTemplateMetaData addMatchingTemplate(Consumer configurator) { diff --git a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java index cf0c766972e09..67c9c3a7844bb 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java @@ -77,7 +77,6 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { assertAcked( client().admin().indices().prepareCreate(indexName) .setSettings(Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) .put(IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.getKey(), 1.0f) @@ -206,7 +205,6 @@ public void testFullClusterRestartPerformNoopRecovery() throws Exception { assertAcked( client().admin().indices().prepareCreate(indexName) .setSettings(Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb") .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) @@ -243,7 +241,6 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception { assertAcked( client().admin().indices().prepareCreate(indexName) .setSettings(Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb") .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) @@ -324,7 +321,6 @@ public void testPeerRecoveryForClosedIndices() throws Exception { createIndex(indexName, Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms") .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms") .build()); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 1a19647ea7f78..57dfebbd231c0 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.translog.Translog; @@ -572,38 +571,4 @@ public void testIgnoreTranslogRetentionSettingsIfSoftDeletesEnabled() { assertThat(indexSettings.getTranslogRetentionAge().millis(), equalTo(-1L)); assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L)); } - - public void testUpdateTranslogRetentionSettingsWithSoftDeletesDisabled() { - Settings.Builder settings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); - - TimeValue ageSetting = TimeValue.timeValueHours(12); - if (randomBoolean()) { - ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000)); - settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting); - } - ByteSizeValue sizeSetting = new ByteSizeValue(512, ByteSizeUnit.MB); - if (randomBoolean()) { - sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024)); - settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting); - } - IndexMetaData metaData = newIndexMeta("index", settings.build()); - IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY); - assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); - assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); - - Settings.Builder newSettings = Settings.builder().put(settings.build()); - if (randomBoolean()) { - ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000)); - newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting); - } - if (randomBoolean()) { - sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024)); - newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting); - } - indexSettings.updateIndexMetaData(newIndexMeta("index", newSettings.build())); - assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); - assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); - } } diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 2d5dc675f025b..bcf20e1fd1f0b 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -5477,8 +5477,9 @@ public long softUpdateDocuments(Term term, Iterable operations = generateHistoryOnReplica(between(1, 500), randomBoolean(), randomBoolean(), randomBoolean()); final IndexMetaData indexMetaData = IndexMetaData.builder(defaultSettings.getIndexMetaData()) - .settings(Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)) - .build(); + .settings(Settings.builder().put(defaultSettings.getSettings()) + .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0)) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)).build(); final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData); try (Store store = createStore()) { EngineConfig config = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get); diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java index 53a96e531585a..903df05979e78 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java @@ -109,8 +109,7 @@ public void testIndexingDuringFileRecovery() throws Exception { } public void testRecoveryOfDisconnectedReplica() throws Exception { - Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build(); - try (ReplicationGroup shards = createGroup(1, settings)) { + try (ReplicationGroup shards = createGroup(1)) { shards.startAll(); int docs = shards.indexDocs(randomInt(50)); shards.flush(); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 86fa8c613b628..4f90b042ebd10 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -844,7 +844,6 @@ public void testLimitNumberOfRetainedTranslogFiles() throws Exception { Settings.Builder settings = Settings.builder() .put(SETTING_NUMBER_OF_SHARDS, 1) .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) .put(IndexSettings.INDEX_TRANSLOG_RETENTION_TOTAL_FILES_SETTING.getKey(), translogRetentionTotalFiles); if (randomBoolean()) { settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), new ByteSizeValue(between(1, 1024 * 1024))); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java index 31bdfce261ad9..28995efc4be20 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java @@ -303,21 +303,6 @@ public void testRetentionLeaseStats() throws IOException { } } - public void testRetentionLeasesActionsFailWithSoftDeletesDisabled() throws Exception { - IndexShard shard = newStartedShard(true, Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); - assertThat(expectThrows(AssertionError.class, () -> shard.addRetentionLease(randomAlphaOfLength(10), - randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE), "test", ActionListener.wrap(() -> {}))).getMessage(), - equalTo("retention leases requires soft deletes but [index] does not have soft deletes enabled")); - assertThat(expectThrows(AssertionError.class, () -> shard.renewRetentionLease( - randomAlphaOfLength(10), randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE), "test")).getMessage(), - equalTo("retention leases requires soft deletes but [index] does not have soft deletes enabled")); - assertThat(expectThrows(AssertionError.class, () -> shard.removeRetentionLease( - randomAlphaOfLength(10), ActionListener.wrap(() -> {}))).getMessage(), - equalTo("retention leases requires soft deletes but [index] does not have soft deletes enabled")); - shard.syncRetentionLeases(); - closeShards(shard); - } - private void assertRetentionLeases( final IndexShard indexShard, final int size, diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 6a8a63ce17f32..39f120b2fa3a3 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2113,8 +2113,7 @@ public void testPrimaryHandOffUpdatesLocalCheckpoint() throws IOException { } IndexShardTestCase.updateRoutingEntry(primarySource, primarySource.routingEntry().relocate(randomAlphaOfLength(10), -1)); - final IndexShard primaryTarget = newShard(primarySource.routingEntry().getTargetRelocatingShard(), Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), primarySource.indexSettings().isSoftDeleteEnabled()).build()); + final IndexShard primaryTarget = newShard(primarySource.routingEntry().getTargetRelocatingShard()); updateMappings(primaryTarget, primarySource.indexSettings().getIndexMetaData()); recoverReplica(primaryTarget, primarySource, true); @@ -2128,13 +2127,11 @@ public void testPrimaryHandOffUpdatesLocalCheckpoint() throws IOException { /* This test just verifies that we fill up local checkpoint up to max seen seqID on primary recovery */ public void testRecoverFromStoreWithNoOps() throws IOException { - final Settings settings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build(); - final IndexShard shard = newStartedShard(true, settings); + final IndexShard shard = newStartedShard(true); indexDoc(shard, "_doc", "0"); indexDoc(shard, "_doc", "1"); // start a replica shard and index the second doc - final IndexShard otherShard = newStartedShard(false, settings); + final IndexShard otherShard = newStartedShard(false); updateMappings(otherShard, shard.indexSettings().getIndexMetaData()); SourceToParse sourceToParse = new SourceToParse(shard.shardId().getIndexName(), "1", new BytesArray("{}"), XContentType.JSON); @@ -2314,8 +2311,7 @@ public void testRecoveryFailsAfterMovingToRelocatedState() throws InterruptedExc public void testRestoreShard() throws IOException { final IndexShard source = newStartedShard(true); - IndexShard target = newStartedShard(true, Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), source.indexSettings().isSoftDeleteEnabled()).build()); + IndexShard target = newStartedShard(true); indexDoc(source, "_doc", "0"); EngineTestCase.generateNewSeqNo(source.getEngine()); // create a gap in the history diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 999028cb083fd..8ddb59ef5db1a 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -446,7 +446,6 @@ public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOE final IndexMetaData.Builder indexMetaData = IndexMetaData.builder("test").settings(Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, between(0,5)) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1,5)) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random())) .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))); if (randomBoolean()) { diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java index 05aebcc459a6e..5cadc31d7946e 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java @@ -28,6 +28,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.bulk.BulkShardRequest; @@ -55,6 +56,7 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.SnapshotMatchers; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.HashMap; @@ -132,7 +134,9 @@ public void testRecoveryWithOutOfOrderDeleteWithTranslog() throws Exception { * - index #5 * - If flush and the translog retention disabled, delete #1 will be removed while index #0 is still retained and replayed. */ - Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build(); + Settings settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0)) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build(); try (ReplicationGroup shards = createGroup(1, settings)) { shards.startAll(); // create out of order delete and index op on replica diff --git a/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java index c95b61f7e6a90..df908fd4f8cce 100644 --- a/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java +++ b/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -354,8 +353,6 @@ public void testTranslogStats() throws Exception { createIndex(indexName, Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .build()); - boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get( - client().admin().indices().prepareGetSettings(indexName).get().getIndexToSettings().get(indexName)); final int nbDocs = randomIntBetween(0, 50); int uncommittedOps = 0; @@ -375,8 +372,8 @@ public void testTranslogStats() throws Exception { assertBusy(() -> { IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get(); assertThat(stats.getIndex(indexName), notNullValue()); - assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo( - softDeletesEnabled ? uncommittedTranslogOps : nbDocs)); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), + equalTo(uncommittedTranslogOps)); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedTranslogOps)); }); @@ -386,8 +383,7 @@ public void testTranslogStats() throws Exception { IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).setIndicesOptions(indicesOptions) .clear().setTranslog(true).get(); assertThat(stats.getIndex(indexName), notNullValue()); - assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), - equalTo(softDeletesEnabled ? 0 : nbDocs)); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(0)); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0)); } } diff --git a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 7a3de4f4c977e..2a29582fdefd3 100644 --- a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -47,7 +47,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLease; import org.elasticsearch.index.shard.IndexEventListener; @@ -627,7 +626,6 @@ public void testRelocationEstablishedPeerRecoveryRetentionLeases() throws Except ensureStableCluster(halfNodes * 2); assertAcked( client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, halfNodes - 1)) .put("index.routing.allocation.include.color", "blue"))); ensureGreen("test"); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 6ec344783c8f2..f2a44b766e73e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -184,9 +184,7 @@ protected Settings indexSettings() { .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(), between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY))) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000)) + .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)) .build(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index c067c0717dcce..54bc1c3dfcda2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -133,9 +133,7 @@ protected IndexMetaData buildIndexMetaData(int replicas, Settings indexSettings, Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000)) + .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)) .put(indexSettings) .build(); IndexMetaData.Builder metaData = IndexMetaData.builder(index.getName()) diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index d0c5bf1ca53a1..c215078635915 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -249,7 +249,6 @@ protected IndexShard newShard( Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000)) .put(settings) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 038efa4d4159f..dd4f937039afc 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -670,7 +670,6 @@ public Settings indexSettings() { } // always default delayed allocation to 0 to make sure we have tests are not delayed builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0); - builder.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); if (randomBoolean()) { builder.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 6265dbe559fa5..79ded94250da6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -95,10 +95,8 @@ protected void startNode(long seed) throws Exception { .preparePutTemplate("random-soft-deletes-template") .setPatterns(Collections.singletonList("*")) .setOrder(0) - .setSettings(Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000)) - ).get(); + .setSettings(Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000))) + .get(); } private static void stopNode() throws IOException, InterruptedException { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index f944dd70813bc..dcbcad3b2048d 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -358,41 +358,6 @@ public void testConflictingPatterns() throws Exception { assertFalse(ESIntegTestCase.indexExists("copy-logs-201801", followerClient())); } - public void testAutoFollowSoftDeletesDisabled() throws Exception { - putAutoFollowPatterns("my-pattern1", new String[] {"logs-*"}); - - // Soft deletes are disabled: - Settings leaderIndexSettings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) - .build(); - createLeaderIndex("logs-20200101", leaderIndexSettings); - assertBusy(() -> { - AutoFollowStats autoFollowStats = getAutoFollowStats(); - assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(0L)); - assertThat(autoFollowStats.getNumberOfFailedFollowIndices(), equalTo(1L)); - assertThat(autoFollowStats.getRecentAutoFollowErrors().size(), equalTo(1)); - ElasticsearchException failure = autoFollowStats.getRecentAutoFollowErrors().firstEntry().getValue().v2(); - assertThat(failure.getMessage(), equalTo("index [logs-20200101] cannot be followed, " + - "because soft deletes are not enabled")); - assertFalse(ESIntegTestCase.indexExists("copy-logs-20200101", followerClient())); - }); - - // Soft deletes are enabled: - leaderIndexSettings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) - .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) - .build(); - createLeaderIndex("logs-20200102", leaderIndexSettings); - assertBusy(() -> { - AutoFollowStats autoFollowStats = getAutoFollowStats(); - assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(1L)); - assertTrue(ESIntegTestCase.indexExists("copy-logs-20200102", followerClient())); - }); - } - public void testPauseAndResumeAutoFollowPattern() throws Exception { final Settings leaderIndexSettings = Settings.builder() .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java index 3a52950d1788b..b623be7ccbe0c 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.CcrSingleNodeTestCase; import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction; @@ -78,21 +77,6 @@ public void testFollowIndex() throws Exception { ensureEmptyWriteBuffers(); } - public void testDoNotCreateFollowerIfLeaderDoesNotHaveSoftDeletes() throws Exception { - final String leaderIndexSettings = getIndexSettings(2, 0, - singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "false")); - assertAcked(client().admin().indices().prepareCreate("leader-index").setSource(leaderIndexSettings, XContentType.JSON)); - ResumeFollowAction.Request followRequest = getResumeFollowRequest("follower"); - followRequest.setFollowerIndex("follower-index"); - PutFollowAction.Request putFollowRequest = getPutFollowRequest("leader", "follower"); - putFollowRequest.setLeaderIndex("leader-index"); - putFollowRequest.setFollowerIndex("follower-index"); - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, - () -> client().execute(PutFollowAction.INSTANCE, putFollowRequest).actionGet()); - assertThat(error.getMessage(), equalTo("leader index [leader-index] does not have soft deletes enabled")); - assertThat(ESIntegTestCase.indexExists("follower-index", client()), equalTo(false)); - } - public void testRemoveRemoteConnection() throws Exception { PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); request.setName("my_pattern"); From 66c507d38fbb2c456d146e8b9be7968d09f792ff Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Jan 2020 17:05:08 -0500 Subject: [PATCH 2/8] wording --- docs/reference/migration/migrate_8_0/indices.asciidoc | 5 +++-- .../cluster/metadata/MetaDataCreateIndexService.java | 2 +- .../cluster/metadata/MetaDataCreateIndexServiceTests.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/reference/migration/migrate_8_0/indices.asciidoc b/docs/reference/migration/migrate_8_0/indices.asciidoc index 64bc11f9c79fa..03acac7753539 100644 --- a/docs/reference/migration/migrate_8_0/indices.asciidoc +++ b/docs/reference/migration/migrate_8_0/indices.asciidoc @@ -40,5 +40,6 @@ instead as it has the same effect as a synced flush in 7.6 and later. ==== Indices with soft deletes disabled Creating indices with soft deletes disabled was deprecated in 7.6 and -is no longer supported in 8.0. Do not specify value for the setting -`[index.soft_deletes.enabled]`. +is no longer supported in 8.0. The setting index.soft_deletes.enabled +can no longer be set to false. As the setting defaults to true, simply +leave the setting unset. diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 662f69763f2f2..c5ecd1d6eebb9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -437,7 +437,7 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState); if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) == false) { throw new IllegalArgumentException("Creating indices with soft-deletes disabled is no longer supported. " + - "Please do not specify value for setting [index.soft_deletes.enabled]."); + "Please do not specify a value for setting [index.soft_deletes.enabled]."); } return indexSettings; } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 07f53e3025e10..416491ce31ba5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -834,7 +834,7 @@ public void testRejectWithSoftDeletesDisabled() { null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); }); assertThat(error.getMessage(), equalTo("Creating indices with soft-deletes disabled is no longer supported. " - + "Please do not specify value for setting [index.soft_deletes.enabled].")); + + "Please do not specify a value for setting [index.soft_deletes.enabled].")); } private IndexTemplateMetaData addMatchingTemplate(Consumer configurator) { From 1c08c1da683f9c2d122c428b96f83f68f9a73399 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Jan 2020 17:22:05 -0500 Subject: [PATCH 3/8] fix the test --- .../RecoveryDuringReplicationTests.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java index 903df05979e78..fa86cc8092a0f 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java @@ -30,6 +30,8 @@ import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -135,24 +137,23 @@ public void testRecoveryOfDisconnectedReplica() throws Exception { final int missingOnReplica = shards.indexDocs(randomInt(5)); docs += missingOnReplica; - final boolean translogTrimmed; + final boolean peerRecoveryRetentionLeaseExpired; if (randomBoolean()) { - shards.flush(); - translogTrimmed = randomBoolean(); - if (translogTrimmed) { - final Translog translog = getTranslog(shards.getPrimary()); - translog.getDeletionPolicy().setRetentionAgeInMillis(0); - translog.trimUnreferencedReaders(); + peerRecoveryRetentionLeaseExpired = randomBoolean(); + if (peerRecoveryRetentionLeaseExpired) { + PlainActionFuture listener = new PlainActionFuture<>(); + shards.getPrimary().removePeerRecoveryRetentionLease(originalReplica.routingEntry().currentNodeId(), listener); + listener.actionGet(); } } else { - translogTrimmed = false; + peerRecoveryRetentionLeaseExpired = false; } originalReplica.close("disconnected", false); IOUtils.close(originalReplica.store()); final IndexShard recoveredReplica = shards.addReplicaWithExistingPath(originalReplica.shardPath(), originalReplica.routingEntry().currentNodeId()); shards.recoverReplica(recoveredReplica); - if (translogTrimmed && missingOnReplica > 0) { + if (peerRecoveryRetentionLeaseExpired && missingOnReplica > 0) { // replica has something to catch up with, but since we trimmed the primary translog, we should fall back to full recovery assertThat(recoveredReplica.recoveryState().getIndex().fileDetails(), not(empty())); } else { From 0ecee877a4c92beef7bfac4735e68056555b48e3 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Jan 2020 21:47:35 -0500 Subject: [PATCH 4/8] resize without soft-deletes --- .../upgrades/FullClusterRestartIT.java | 79 +++++++++++++++++++ .../indices/shrink/TransportResizeAction.java | 4 +- .../metadata/MetaDataCreateIndexService.java | 3 +- .../MetaDataCreateIndexServiceTests.java | 3 +- 4 files changed, 86 insertions(+), 3 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 4e609943f0ba8..bc97f9c560678 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -279,6 +279,9 @@ public void testShrink() throws IOException { mappingsAndSettings.startObject("settings"); { mappingsAndSettings.field("index.number_of_shards", 5); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + mappingsAndSettings.field(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), Boolean.toString(randomBoolean())); + } } mappingsAndSettings.endObject(); } @@ -350,6 +353,9 @@ public void testShrinkAfterUpgrade() throws IOException { // the default number of shards is now one so we have to set the number of shards to be more than one explicitly mappingsAndSettings.startObject("settings"); mappingsAndSettings.field("index.number_of_shards", 5); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + mappingsAndSettings.field(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), Boolean.toString(randomBoolean())); + } mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); @@ -1310,4 +1316,77 @@ public void testTurnOffTranslogRetentionAfterUpgraded() throws Exception { ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index); } } + + public void testResize() throws Exception { + int numDocs; + if (isRunningAgainstOldCluster()) { + final Settings.Builder settings = Settings.builder() + .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3) + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false); + } + createIndex(index, settings.build()); + numDocs = randomIntBetween(10, 1000); + for (int i = 0; i < numDocs; i++) { + indexDocument(Integer.toString(i)); + if (rarely()) { + flush(index, randomBoolean()); + } + } + saveInfoDocument("num_doc_" + index, Integer.toString(numDocs)); + ensureGreen(index); + } else { + ensureGreen(index); + numDocs = Integer.parseInt(loadInfoDocument("num_doc_" + index)); + int moreDocs = randomIntBetween(0, 100); + for (int i = 0; i < moreDocs; i++) { + indexDocument(Integer.toString(numDocs + i)); + if (rarely()) { + flush(index, randomBoolean()); + } + } + Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings"); + updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}"); + client().performRequest(updateSettingsRequest); + { + final String target = index + "_shrunken"; + Request shrinkRequest = new Request("PUT", "/" + index + "/_shrink/" + target); + Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1); + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); + } + shrinkRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}"); + client().performRequest(shrinkRequest); + ensureGreenLongWait(target); + assertNumHits(target, numDocs + moreDocs, 1); + } + { + final String target = index + "_split"; + Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 6); + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); + } + Request splitRequest = new Request("PUT", "/" + index + "/_split/" + target); + splitRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}"); + client().performRequest(splitRequest); + ensureGreenLongWait(target); + assertNumHits(target, numDocs + moreDocs, 6); + } + { + final String target = index + "_cloned"; + client().performRequest(new Request("PUT", "/" + index + "/_clone/" + target)); + ensureGreenLongWait(target); + assertNumHits(target, numDocs + moreDocs, 3); + } + } + } + + private void assertNumHits(String index, int numHits, int totalShards) throws IOException { + Map resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))); + assertNoFailures(resp); + assertThat(XContentMapValues.extractValue("_shards.total", resp), equalTo(totalShards)); + assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards)); + assertThat(extractTotalHits(resp), equalTo(numHits)); + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 2d1a3964819c8..f13c1096d5548 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -175,7 +175,9 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize"); } } - if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(targetIndexSettings) == false) { + if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(metaData.getSettings()) && + IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(targetIndexSettings) && + IndexSettings.INDEX_SOFT_DELETES_SETTING.get(targetIndexSettings) == false) { throw new IllegalArgumentException("Can't disable [index.soft_deletes.enabled] setting on resize"); } String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index"; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index c5ecd1d6eebb9..c946c8edebd49 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -435,7 +435,8 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu * that will be used to create this index. */ MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState); - if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) == false) { + if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) == false + && IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.V_8_0_0)) { throw new IllegalArgumentException("Creating indices with soft-deletes disabled is no longer supported. " + "Please do not specify a value for setting [index.soft_deletes.enabled]."); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 416491ce31ba5..ef814f857accf 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -831,7 +831,8 @@ public void testRejectWithSoftDeletesDisabled() { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), - null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + null, Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT).build(), + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); }); assertThat(error.getMessage(), equalTo("Creating indices with soft-deletes disabled is no longer supported. " + "Please do not specify a value for setting [index.soft_deletes.enabled].")); From 9a992a9aff741adc98e494eed9814769e7e21dbc Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Jan 2020 21:49:50 -0500 Subject: [PATCH 5/8] snapshot/restore without soft-deletes --- .../org/elasticsearch/upgrades/FullClusterRestartIT.java | 5 +++++ .../java/org/elasticsearch/snapshots/RestoreService.java | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index bc97f9c560678..9e77eb2ff03cb 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -782,6 +782,11 @@ public void testSnapshotRestore() throws IOException { if (isRunningAgainstOldCluster()) { // Create the index count = between(200, 300); + Settings.Builder settings = Settings.builder(); + if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } + createIndex(index, settings.build()); indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject()); } else { count = countOfIndexedRandomDocuments(); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 3a14789a68152..84c08e02bf179 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -65,6 +65,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.repositories.IndexId; @@ -123,7 +124,8 @@ public class RestoreService implements ClusterStateApplier { SETTING_NUMBER_OF_SHARDS, SETTING_VERSION_CREATED, SETTING_INDEX_UUID, - SETTING_CREATION_DATE)); + SETTING_CREATION_DATE, + IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey())); // It's OK to change some settings, but we shouldn't allow simply removing them private static final Set UNREMOVABLE_SETTINGS; From 014b558228d5164cd200b79207676ccc2b51b437 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Jan 2020 22:13:26 -0500 Subject: [PATCH 6/8] less random settings --- .../org/elasticsearch/upgrades/FullClusterRestartIT.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 9e77eb2ff03cb..406099c82e09e 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -279,9 +279,6 @@ public void testShrink() throws IOException { mappingsAndSettings.startObject("settings"); { mappingsAndSettings.field("index.number_of_shards", 5); - if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { - mappingsAndSettings.field(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), Boolean.toString(randomBoolean())); - } } mappingsAndSettings.endObject(); } @@ -353,9 +350,6 @@ public void testShrinkAfterUpgrade() throws IOException { // the default number of shards is now one so we have to set the number of shards to be more than one explicitly mappingsAndSettings.startObject("settings"); mappingsAndSettings.field("index.number_of_shards", 5); - if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) { - mappingsAndSettings.field(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), Boolean.toString(randomBoolean())); - } mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); From a794188d69e9d96ef7ded89b156d4c396e17eaee Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Jan 2020 22:49:20 -0500 Subject: [PATCH 7/8] fix settings --- .../cluster/metadata/MetaDataCreateIndexService.java | 2 +- .../cluster/metadata/MetaDataCreateIndexServiceTests.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index c946c8edebd49..f605ee9de75eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -436,7 +436,7 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu */ MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState); if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) == false - && IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.V_8_0_0)) { + && IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(indexSettings).onOrAfter(Version.V_8_0_0)) { throw new IllegalArgumentException("Creating indices with soft-deletes disabled is no longer supported. " + "Please do not specify a value for setting [index.soft_deletes.enabled]."); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index ef814f857accf..416491ce31ba5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -831,8 +831,7 @@ public void testRejectWithSoftDeletesDisabled() { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), - null, Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT).build(), - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); }); assertThat(error.getMessage(), equalTo("Creating indices with soft-deletes disabled is no longer supported. " + "Please do not specify a value for setting [index.soft_deletes.enabled].")); From f1c09e7a7cc147d18cb176275bc35b5c91c01edc Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 16 Jan 2020 23:34:35 -0500 Subject: [PATCH 8/8] Remove translog test --- .../RecoveryDuringReplicationTests.java | 63 ------------------- 1 file changed, 63 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java index fa86cc8092a0f..ec10e2a5e8444 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java @@ -30,8 +30,6 @@ import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -40,7 +38,6 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.DocIdSeqNoAndSource; @@ -110,66 +107,6 @@ public void testIndexingDuringFileRecovery() throws Exception { } } - public void testRecoveryOfDisconnectedReplica() throws Exception { - try (ReplicationGroup shards = createGroup(1)) { - shards.startAll(); - int docs = shards.indexDocs(randomInt(50)); - shards.flush(); - final IndexShard originalReplica = shards.getReplicas().get(0); - for (int i = 0; i < randomInt(2); i++) { - final int indexedDocs = shards.indexDocs(randomInt(5)); - docs += indexedDocs; - - final boolean flush = randomBoolean(); - if (flush) { - originalReplica.flush(new FlushRequest()); - } - } - - // simulate a background global checkpoint sync at which point we expect the global checkpoint to advance on the replicas - shards.syncGlobalCheckpoint(); - long globalCheckpointOnReplica = originalReplica.getLastSyncedGlobalCheckpoint(); - Optional safeCommitOnReplica = - originalReplica.store().findSafeIndexCommit(globalCheckpointOnReplica); - assertTrue(safeCommitOnReplica.isPresent()); - shards.removeReplica(originalReplica); - - final int missingOnReplica = shards.indexDocs(randomInt(5)); - docs += missingOnReplica; - - final boolean peerRecoveryRetentionLeaseExpired; - if (randomBoolean()) { - peerRecoveryRetentionLeaseExpired = randomBoolean(); - if (peerRecoveryRetentionLeaseExpired) { - PlainActionFuture listener = new PlainActionFuture<>(); - shards.getPrimary().removePeerRecoveryRetentionLease(originalReplica.routingEntry().currentNodeId(), listener); - listener.actionGet(); - } - } else { - peerRecoveryRetentionLeaseExpired = false; - } - originalReplica.close("disconnected", false); - IOUtils.close(originalReplica.store()); - final IndexShard recoveredReplica = - shards.addReplicaWithExistingPath(originalReplica.shardPath(), originalReplica.routingEntry().currentNodeId()); - shards.recoverReplica(recoveredReplica); - if (peerRecoveryRetentionLeaseExpired && missingOnReplica > 0) { - // replica has something to catch up with, but since we trimmed the primary translog, we should fall back to full recovery - assertThat(recoveredReplica.recoveryState().getIndex().fileDetails(), not(empty())); - } else { - assertThat(recoveredReplica.recoveryState().getIndex().fileDetails(), empty()); - assertThat(recoveredReplica.recoveryState().getTranslog().recoveredOperations(), - equalTo(Math.toIntExact(docs - 1 - safeCommitOnReplica.get().localCheckpoint))); - assertThat(recoveredReplica.recoveryState().getTranslog().totalLocal(), - equalTo(Math.toIntExact(globalCheckpointOnReplica - safeCommitOnReplica.get().localCheckpoint))); - } - - docs += shards.indexDocs(randomInt(5)); - - shards.assertAllEqual(docs); - } - } - /* * Simulate a scenario with two replicas where one of the replicas receives an extra document, the other replica is promoted on primary * failure, the receiving replica misses the primary/replica re-sync and then recovers from the primary. We expect that a