From 16b90ee6a8b7eff826a56c72bb1ac52ab36ae6db Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 20 Jan 2021 17:06:45 -0500 Subject: [PATCH 01/13] Drop getAverageSizeInBytes It had no non-test callers, so YAGNI. --- .../elasticsearch/index/shard/DocsStats.java | 8 ------- .../index/engine/NoOpEngineTests.java | 1 - .../index/shard/DocsStatsTests.java | 21 +------------------ .../index/shard/IndexShardTests.java | 4 ++-- 4 files changed, 3 insertions(+), 31 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java b/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java index f757f2ae15aec..ba1d31db257c7 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java @@ -79,14 +79,6 @@ public long getTotalSizeInBytes() { return totalSizeInBytes; } - /** - * Returns the average size in bytes of all documents in this stats. - */ - public long getAverageSizeInBytes() { - long totalDocs = count + deleted; - return totalDocs == 0 ? 0 : totalSizeInBytes / totalDocs; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(count); diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java index c061e7e4a0278..ebb6fe27cf2c6 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java @@ -150,7 +150,6 @@ public void testNoOpEngineStats() throws Exception { assertEquals(expectedDocStats.getCount(), noOpEngine.docStats().getCount()); assertEquals(expectedDocStats.getDeleted(), noOpEngine.docStats().getDeleted()); assertEquals(expectedDocStats.getTotalSizeInBytes(), noOpEngine.docStats().getTotalSizeInBytes()); - assertEquals(expectedDocStats.getAverageSizeInBytes(), noOpEngine.docStats().getAverageSizeInBytes()); assertEquals(expectedSegmentStats.getCount(), noOpEngine.segmentsStats(includeFileSize, true).getCount()); // don't compare memory in bytes since we load the index with term-dict off-heap assertEquals(expectedSegmentStats.getFileSizes().size(), diff --git a/server/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java b/server/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java index 30ced1dd05417..4c46b3fcfe30c 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java @@ -28,36 +28,17 @@ public class DocsStatsTests extends ESTestCase { - public void testCalculateAverageDocSize() throws Exception { - DocsStats stats = new DocsStats(10, 2, 120); - assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); - - stats.add(new DocsStats(0, 0, 0)); - assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); - - stats.add(new DocsStats(8, 30, 480)); - assertThat(stats.getCount(), equalTo(18L)); - assertThat(stats.getDeleted(), equalTo(32L)); - assertThat(stats.getTotalSizeInBytes(), equalTo(600L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(12L)); - } - public void testUninitialisedShards() { DocsStats stats = new DocsStats(0, 0, -1); assertThat(stats.getTotalSizeInBytes(), equalTo(-1L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(0L)); stats.add(new DocsStats(0, 0, -1)); assertThat(stats.getTotalSizeInBytes(), equalTo(-1L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(0L)); stats.add(new DocsStats(1, 0, 10)); assertThat(stats.getTotalSizeInBytes(), equalTo(10L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); stats.add(new DocsStats(0, 0, -1)); assertThat(stats.getTotalSizeInBytes(), equalTo(10L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); stats.add(new DocsStats(1, 0, 20)); assertThat(stats.getTotalSizeInBytes(), equalTo(30L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(15L)); } public void testSerialize() throws Exception { @@ -69,7 +50,7 @@ public void testSerialize() throws Exception { DocsStats cloneStats = new DocsStats(in); assertThat(cloneStats.getCount(), equalTo(originalStats.getCount())); assertThat(cloneStats.getDeleted(), equalTo(originalStats.getDeleted())); - assertThat(cloneStats.getAverageSizeInBytes(), equalTo(originalStats.getAverageSizeInBytes())); + assertThat(cloneStats.getTotalSizeInBytes(), equalTo(originalStats.getTotalSizeInBytes())); } } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 165af70e50da7..e367f2c65ebd4 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2881,7 +2881,7 @@ public void testDocStats() throws Exception { assertTrue(searcher.getIndexReader().numDocs() <= docsStats.getCount()); } assertThat(docsStats.getDeleted(), equalTo(0L)); - assertThat(docsStats.getAverageSizeInBytes(), greaterThan(0L)); + assertThat(docsStats.getTotalSizeInBytes(), greaterThan(0L)); } final List ids = randomSubsetOf( @@ -2941,7 +2941,7 @@ public void testDocStats() throws Exception { final DocsStats docStats = indexShard.docStats(); assertThat(docStats.getCount(), equalTo(numDocs)); assertThat(docStats.getDeleted(), equalTo(0L)); - assertThat(docStats.getAverageSizeInBytes(), greaterThan(0L)); + assertThat(docStats.getTotalSizeInBytes(), greaterThan(0L)); } } finally { closeShards(indexShard); From 02ded439da0a854f3bef94fc3170f5c42b2f224f Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 20 Jan 2021 17:06:47 -0500 Subject: [PATCH 02/13] Refactor TransportRolloverAction a bit to separate the building of a Conditition.Stats cleanly from the evaluation of same. --- .../rollover/TransportRolloverAction.java | 40 ++++++++------- .../TransportRolloverActionTests.java | 50 ++++++++----------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index 2dafd7839a468..ee365e369f1c5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -51,6 +51,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -106,7 +107,7 @@ protected void masterOperation(Task task, final RolloverRequest rolloverRequest, @Override public void onResponse(IndicesStatsResponse statsResponse) { final Map conditionResults = evaluateConditions(rolloverRequest.getConditions().values(), - metadata.index(sourceIndexName), statsResponse); + buildStats(metadata.index(sourceIndexName), statsResponse)); if (rolloverRequest.isDryRun()) { listener.onResponse( @@ -167,30 +168,35 @@ public void onFailure(Exception e) { } static Map evaluateConditions(final Collection> conditions, - @Nullable final DocsStats docsStats, - @Nullable final IndexMetadata metadata) { - if (metadata == null) { - return conditions.stream().collect(Collectors.toMap(Condition::toString, cond -> false)); + @Nullable final Condition.Stats stats) { + Objects.requireNonNull(conditions, "conditions must not be null"); + + if (stats != null) { + return conditions.stream() + .map(condition -> condition.evaluate(stats)) + .collect(Collectors.toMap(result -> result.condition.toString(), result -> result.matched)); + } else { + // no conditions matched + return conditions.stream() + .collect(Collectors.toMap(Condition::toString, cond -> false)); } - final long numDocs = docsStats == null ? 0 : docsStats.getCount(); - final long indexSize = docsStats == null ? 0 : docsStats.getTotalSizeInBytes(); - final Condition.Stats stats = new Condition.Stats(numDocs, metadata.getCreationDate(), new ByteSizeValue(indexSize)); - return conditions.stream() - .map(condition -> condition.evaluate(stats)) - .collect(Collectors.toMap(result -> result.condition.toString(), result -> result.matched)); } - static Map evaluateConditions(final Collection> conditions, - @Nullable final IndexMetadata metadata, - @Nullable final IndicesStatsResponse statsResponse) { + static Condition.Stats buildStats(@Nullable final IndexMetadata metadata, + @Nullable final IndicesStatsResponse statsResponse) { if (metadata == null) { - return conditions.stream().collect(Collectors.toMap(Condition::toString, cond -> false)); + return null; } else { final DocsStats docsStats = Optional.ofNullable(statsResponse) .map(stats -> stats.getIndex(metadata.getIndex().getName())) - .map(indexStats -> indexStats.getPrimaries().getDocs()) + .map(stats -> stats.getPrimaries().getDocs()) .orElse(null); - return evaluateConditions(conditions, docsStats, metadata); + + return new Condition.Stats( + docsStats == null ? 0 : docsStats.getCount(), + metadata.getCreationDate(), + new ByteSizeValue(docsStats == null ? 0 : docsStats.getTotalSizeInBytes()) + ); } } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 34e9f90f9ae4d..9b2cc6af33a03 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -80,6 +80,7 @@ import java.util.Set; import static java.util.Collections.emptyList; +import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.buildStats; import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.evaluateConditions; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -100,8 +101,8 @@ public void testDocStatsSelectionFromPrimariesOnly() { final Condition condition = createTestCondition(); String indexName = randomAlphaOfLengthBetween(5, 7); - evaluateConditions(Sets.newHashSet(condition), createMetadata(indexName), - createIndicesStatResponse(indexName, docsInShards, docsInPrimaryShards)); + evaluateConditions(Sets.newHashSet(condition), + buildStats(createMetadata(indexName), createIndicesStatResponse(indexName, docsInShards, docsInPrimaryShards))); final ArgumentCaptor argument = ArgumentCaptor.forClass(Condition.Stats.class); verify(condition).evaluate(argument.capture()); @@ -112,29 +113,28 @@ public void testEvaluateConditions() { MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2)); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB)); + final Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); long matchMaxDocs = randomIntBetween(100, 1000); long notMatchMaxDocs = randomIntBetween(0, 99); ByteSizeValue notMatchMaxSize = new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB); - final Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - final IndexMetadata metadata = IndexMetadata.builder(randomAlphaOfLength(10)) - .creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(3).getMillis()) - .settings(settings) - .build(); - final Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); - Map results = evaluateConditions(conditions, - new DocsStats(matchMaxDocs, 0L, ByteSizeUnit.MB.toBytes(120)), metadata); + long indexCreated = TimeValue.timeValueHours(3).getMillis(); + + expectThrows(NullPointerException.class, () -> evaluateConditions(null, new Condition.Stats(0, 0, ByteSizeValue.ofMb(0)))); + + Map results = evaluateConditions(conditions, null); + assertThat(results.size(), equalTo(3)); + for (Boolean matched : results.values()) { + assertThat(matched, equalTo(false)); + } + + results = evaluateConditions(conditions, new Condition.Stats(matchMaxDocs, indexCreated, ByteSizeValue.ofMb(120))); assertThat(results.size(), equalTo(3)); for (Boolean matched : results.values()) { assertThat(matched, equalTo(true)); } - results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0, notMatchMaxSize.getBytes()), metadata); + results = evaluateConditions(conditions, new Condition.Stats(notMatchMaxDocs, indexCreated, notMatchMaxSize)); assertThat(results.size(), equalTo(3)); for (Map.Entry entry : results.entrySet()) { if (entry.getKey().equals(maxAgeCondition.toString())) { @@ -149,12 +149,12 @@ public void testEvaluateConditions() { } } - public void testEvaluateWithoutDocStats() { + public void testEvaluateWithoutStats() { MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomNonNegativeLong()); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(randomIntBetween(1, 3))); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())); + final Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); - Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) @@ -166,7 +166,7 @@ public void testEvaluateWithoutDocStats() { .creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(randomIntBetween(5, 10)).getMillis()) .settings(settings) .build(); - Map results = evaluateConditions(conditions, null, metadata); + Map results = evaluateConditions(conditions, buildStats(metadata, null)); assertThat(results.size(), equalTo(3)); for (Map.Entry entry : results.entrySet()) { @@ -186,13 +186,7 @@ public void testEvaluateWithoutMetadata() { MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2)); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB)); - - long matchMaxDocs = randomIntBetween(100, 1000); final Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); - Map results = evaluateConditions(conditions, - new DocsStats(matchMaxDocs, 0L, ByteSizeUnit.MB.toBytes(120)), null); - assertThat(results.size(), equalTo(3)); - results.forEach((k, v) -> assertFalse(v)); final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) @@ -206,9 +200,9 @@ public void testEvaluateWithoutMetadata() { .settings(settings) .build(); IndicesStatsResponse indicesStats = randomIndicesStatsResponse(new IndexMetadata[]{metadata}); - Map results2 = evaluateConditions(conditions, null, indicesStats); - assertThat(results2.size(), equalTo(3)); - results2.forEach((k, v) -> assertFalse(v)); + Map results = evaluateConditions(conditions, buildStats(null, indicesStats)); + assertThat(results.size(), equalTo(3)); + results.forEach((k, v) -> assertFalse(v)); } From 0c8498b52b0df8a760626537b438004b542c5bbe Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 20 Jan 2021 17:06:48 -0500 Subject: [PATCH 03/13] Whitespace --- .../client/indices/rollover/RolloverRequest.java | 1 + .../action/admin/indices/rollover/RolloverIT.java | 2 +- .../admin/indices/rollover/RolloverRequest.java | 7 ++++--- .../org/elasticsearch/indices/IndicesModule.java | 6 +++--- .../admin/indices/rollover/ConditionTests.java | 4 ++-- .../rollover/TransportRolloverActionTests.java | 13 ++++++------- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java index ef78fb7353067..6383c0fd88d5f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java @@ -105,6 +105,7 @@ public RolloverRequest addMaxIndexDocsCondition(long numDocs) { this.conditions.put(maxDocsCondition.name(), maxDocsCondition); return this; } + /** * Adds a size-based condition to check if the index size is at least size. */ diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java index 9a9dff3588fea..276ab95130959 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java @@ -197,7 +197,7 @@ public void testRolloverWithIndexSettingsWithoutPrefix() throws Exception { indexDoc("test_index-2", "1", "field", "value"); flush("test_index-2"); final Settings settings = Settings.builder() - .put("number_of_shards", 1) + .put("number_of_shards", 1) .put("number_of_replicas", 0) .build(); final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias") diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 7283d3c2375aa..094156d2ecb6a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -58,12 +58,12 @@ public class RolloverRequest extends AcknowledgedRequest implem static { CONDITION_PARSER.declareString((conditions, s) -> conditions.put(MaxAgeCondition.NAME, new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))), - MAX_AGE_CONDITION); + MAX_AGE_CONDITION); CONDITION_PARSER.declareLong((conditions, value) -> - conditions.put(MaxDocsCondition.NAME, new MaxDocsCondition(value)), MAX_DOCS_CONDITION); + conditions.put(MaxDocsCondition.NAME, new MaxDocsCondition(value)), MAX_DOCS_CONDITION); CONDITION_PARSER.declareString((conditions, s) -> conditions.put(MaxSizeCondition.NAME, new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))), - MAX_SIZE_CONDITION); + MAX_SIZE_CONDITION); PARSER.declareField((parser, request, context) -> CONDITION_PARSER.parse(parser, request.conditions, null), CONDITIONS, ObjectParser.ValueType.OBJECT); @@ -155,6 +155,7 @@ public void setRolloverTarget(String rolloverTarget) { public void setNewIndexName(String newIndexName) { this.newIndexName = newIndexName; } + /** * Sets if the rollover should not be executed when conditions are met */ diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index e2470c68df027..dfa9914e74963 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -85,9 +85,9 @@ public IndicesModule(List mapperPlugins) { public static List getNamedWriteables() { return Arrays.asList( - new NamedWriteableRegistry.Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new), - new NamedWriteableRegistry.Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new), - new NamedWriteableRegistry.Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new)); + new NamedWriteableRegistry.Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new), + new NamedWriteableRegistry.Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new), + new NamedWriteableRegistry.Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new)); } public static List getNamedXContents() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java index bc6e2d1b610ad..570326352645c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java @@ -76,11 +76,11 @@ public void testMaxSize() { public void testEqualsAndHashCode() { MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomLong()); EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxDocsCondition, condition -> new MaxDocsCondition(condition.value), - condition -> new MaxDocsCondition(randomLong())); + condition -> new MaxDocsCondition(randomLong())); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(randomByteSize()); EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxSizeCondition, condition -> new MaxSizeCondition(condition.value), - condition -> new MaxSizeCondition(randomByteSize())); + condition -> new MaxSizeCondition(randomByteSize())); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(randomNonNegativeLong())); EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxAgeCondition, condition -> new MaxAgeCondition(condition.value), diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 9b2cc6af33a03..94461f29c2014 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -205,7 +205,6 @@ public void testEvaluateWithoutMetadata() { results.forEach((k, v) -> assertFalse(v)); } - public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPrimariesFromWriteIndex() throws Exception { final TransportService mockTransportService = mock(TransportService.class); final ClusterService mockClusterService = mock(ClusterService.class); @@ -238,20 +237,20 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr assert statsResponse.getTotal().getDocs().getCount() == (total + total); final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("logs-index-000001") - .putAlias(AliasMetadata.builder("logs-alias").writeIndex(false).build()).settings(settings(Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(1); + .putAlias(AliasMetadata.builder("logs-alias").writeIndex(false).build()).settings(settings(Version.CURRENT)) + .numberOfShards(1).numberOfReplicas(1); final IndexMetadata.Builder indexMetadata2 = IndexMetadata.builder("logs-index-000002") - .putAlias(AliasMetadata.builder("logs-alias").writeIndex(true).build()).settings(settings(Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(1); + .putAlias(AliasMetadata.builder("logs-alias").writeIndex(true).build()).settings(settings(Version.CURRENT)) + .numberOfShards(1).numberOfReplicas(1); final ClusterState stateBefore = ClusterState.builder(ClusterName.DEFAULT) - .metadata(Metadata.builder().put(indexMetadata).put(indexMetadata2)).build(); + .metadata(Metadata.builder().put(indexMetadata).put(indexMetadata2)).build(); when(mockCreateIndexService.applyCreateIndexRequest(any(), any(), anyBoolean())).thenReturn(stateBefore); when(mdIndexAliasesService.applyAliasActions(any(), any())).thenReturn(stateBefore); MetadataRolloverService rolloverService = new MetadataRolloverService(mockThreadPool, mockCreateIndexService, mdIndexAliasesService, mockIndexNameExpressionResolver); final TransportRolloverAction transportRolloverAction = new TransportRolloverAction(mockTransportService, mockClusterService, - mockThreadPool, mockActionFilters, mockIndexNameExpressionResolver, rolloverService, mockClient); + mockThreadPool, mockActionFilters, mockIndexNameExpressionResolver, rolloverService, mockClient); // For given alias, verify that condition evaluation fails when the condition doc count is greater than the primaries doc count // (primaries from only write index is considered) From b257bb4a3e80d12494b7f593d94fc460450baa18 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Thu, 21 Jan 2021 13:00:10 -0500 Subject: [PATCH 04/13] Cleanups Reorder things to be age, docs, size more consistently, and a few 'this way, not that way' changes. --- .../rollover/RolloverResponseTests.java | 2 +- .../admin/indices/rollover/ConditionTests.java | 8 ++++---- .../rollover/TransportRolloverActionTests.java | 12 ++++++------ .../cluster/metadata/IndexMetadataTests.java | 10 ++++++---- .../RemoveCorruptedShardDataCommandTests.java | 18 +++++------------- 5 files changed, 22 insertions(+), 28 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java index 78b285493a408..582113873adfd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java @@ -43,8 +43,8 @@ public class RolloverResponseTests extends ESTestCase { private static final List>> conditionSuppliers = new ArrayList<>(); static { conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); } public void testFromXContent() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java index 570326352645c..da51059e2d485 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java @@ -74,6 +74,10 @@ public void testMaxSize() { } public void testEqualsAndHashCode() { + MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(randomNonNegativeLong())); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxAgeCondition, condition -> new MaxAgeCondition(condition.value), + condition -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomLong()); EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxDocsCondition, condition -> new MaxDocsCondition(condition.value), condition -> new MaxDocsCondition(randomLong())); @@ -81,10 +85,6 @@ public void testEqualsAndHashCode() { MaxSizeCondition maxSizeCondition = new MaxSizeCondition(randomByteSize()); EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxSizeCondition, condition -> new MaxSizeCondition(condition.value), condition -> new MaxSizeCondition(randomByteSize())); - - MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(randomNonNegativeLong())); - EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxAgeCondition, condition -> new MaxAgeCondition(condition.value), - condition -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); } private static ByteSizeValue randomByteSize() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 94461f29c2014..0b39ed8ef7e2c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -110,10 +110,10 @@ public void testDocStatsSelectionFromPrimariesOnly() { } public void testEvaluateConditions() { - MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2)); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB)); - final Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); + final Set> conditions = Sets.newHashSet(maxAgeCondition, maxDocsCondition, maxSizeCondition); long matchMaxDocs = randomIntBetween(100, 1000); long notMatchMaxDocs = randomIntBetween(0, 99); @@ -150,10 +150,10 @@ public void testEvaluateConditions() { } public void testEvaluateWithoutStats() { - MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomNonNegativeLong()); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(randomIntBetween(1, 3))); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomNonNegativeLong()); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())); - final Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); + final Set> conditions = Sets.newHashSet(maxAgeCondition, maxDocsCondition, maxSizeCondition); final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) @@ -183,10 +183,10 @@ public void testEvaluateWithoutStats() { } public void testEvaluateWithoutMetadata() { - MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2)); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB)); - final Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); + final Set> conditions = Sets.newHashSet(maxAgeCondition, maxDocsCondition, maxSizeCondition); final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index 402fe7ffe93b0..73512aa35061c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -44,9 +44,9 @@ import org.junit.Before; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -92,9 +92,11 @@ public void testIndexMetadataSerialization() throws IOException { .putCustom("my_custom", customMap) .putRolloverInfo( new RolloverInfo(randomAlphaOfLength(5), - Arrays.asList(new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), - new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())), - new MaxDocsCondition(randomNonNegativeLong())), + List.of( + new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), + new MaxDocsCondition(randomNonNegativeLong()), + new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())) + ), randomNonNegativeLong())).build(); assertEquals(system, metadata.isSystem()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java index 4ef8c3c8d27b0..daddcba73a4eb 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java @@ -126,19 +126,11 @@ public void setup() throws IOException { shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId); // Adding rollover info to IndexMetadata to check that NamedXContentRegistry is properly configured - Condition rolloverCondition; - - switch (randomIntBetween(0, 2)) { - case 0: - rolloverCondition = new MaxDocsCondition(randomNonNegativeLong()); - break; - case 1: - rolloverCondition = new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())); - break; - default: - rolloverCondition = new MaxAgeCondition(new TimeValue(randomNonNegativeLong())); - break; - } + Condition rolloverCondition = randomFrom( + new MaxAgeCondition(new TimeValue(randomNonNegativeLong())), + new MaxDocsCondition(randomNonNegativeLong()), + new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())) + ); final IndexMetadata.Builder metadata = IndexMetadata.builder(routing.getIndexName()) .settings(settings) From 150088ddf43a6790d435a43b4919985dc66fc7e8 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Thu, 21 Jan 2021 13:07:51 -0500 Subject: [PATCH 05/13] Add MaxSinglePrimarySizeCondition --- .../admin/indices/rollover/Condition.java | 13 ++- .../MaxSinglePrimarySizeCondition.java | 83 +++++++++++++++++++ .../indices/rollover/RolloverRequest.java | 29 ++++++- .../rollover/RolloverRequestBuilder.java | 5 ++ .../rollover/TransportRolloverAction.java | 21 ++++- .../elasticsearch/indices/IndicesModule.java | 9 +- .../indices/rollover/ConditionTests.java | 41 ++++++--- .../rollover/RolloverRequestTests.java | 6 +- .../rollover/RolloverResponseTests.java | 3 +- .../TransportRolloverActionTests.java | 45 ++++++---- .../cluster/metadata/IndexMetadataTests.java | 4 +- .../RemoveCorruptedShardDataCommandTests.java | 2 + 12 files changed, 221 insertions(+), 40 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSinglePrimarySizeCondition.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java index 6f1b6a4240b32..89216ea9c326b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.rollover; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContentFragment; @@ -37,6 +38,14 @@ protected Condition(String name) { this.name = name; } + /** + * Checks if this condition is available in a specific version. + * This makes sure BWC when introducing a new condition which is not recognized by older versions. + */ + boolean includedInVersion(Version version) { + return true; + } + public abstract Result evaluate(Stats stats); @Override @@ -77,11 +86,13 @@ public static class Stats { public final long numDocs; public final long indexCreated; public final ByteSizeValue indexSize; + public final ByteSizeValue maxSinglePrimarySize; - public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) { + public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize, ByteSizeValue maxSinglePrimarySize) { this.numDocs = numDocs; this.indexCreated = indexCreated; this.indexSize = indexSize; + this.maxSinglePrimarySize = maxSinglePrimarySize; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSinglePrimarySizeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSinglePrimarySizeCondition.java new file mode 100644 index 0000000000000..28ff2d3bf853e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSinglePrimarySizeCondition.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.rollover; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +/** + * A size-based condition for the primary shards within an index. + * Evaluates to true if the size of the largest primary shard is at least {@link #value}. + */ +public class MaxSinglePrimarySizeCondition extends Condition { + public static final String NAME = "max_single_primary_size"; + + public MaxSinglePrimarySizeCondition(ByteSizeValue value) { + super(NAME); + this.value = value; + } + + public MaxSinglePrimarySizeCondition(StreamInput in) throws IOException { + super(NAME); + this.value = new ByteSizeValue(in.readVLong(), ByteSizeUnit.BYTES); + } + + @Override + public Result evaluate(Stats stats) { + return new Result(this, stats.maxSinglePrimarySize.getBytes() >= value.getBytes()); + } + + @Override + public String getWriteableName() { return NAME; } + + @Override + public void writeTo(StreamOutput out) throws IOException { + // While we technically could serialize this with value.writeTo(...), that would + // require doing the song and dance around backwards compatibility for this value. Since + // in this case the deserialized version is not displayed to a user, it's okay to simply use + // bytes. + out.writeVLong(value.getBytes()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.field(NAME, value.getStringRep()); + } + + public static MaxSinglePrimarySizeCondition fromXContent(XContentParser parser) throws IOException { + if (parser.nextToken() == XContentParser.Token.VALUE_STRING) { + return new MaxSinglePrimarySizeCondition(ByteSizeValue.parseBytesSizeValue(parser.text(), NAME)); + } else { + throw new IllegalArgumentException("invalid token: " + parser.currentToken()); + } + } + + @Override + boolean includedInVersion(Version version) { + return version.onOrAfter(Version.V_8_0_0); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 094156d2ecb6a..72cb594540bb3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -54,16 +55,24 @@ public class RolloverRequest extends AcknowledgedRequest implem private static final ParseField MAX_AGE_CONDITION = new ParseField(MaxAgeCondition.NAME); private static final ParseField MAX_DOCS_CONDITION = new ParseField(MaxDocsCondition.NAME); private static final ParseField MAX_SIZE_CONDITION = new ParseField(MaxSizeCondition.NAME); + private static final ParseField MAX_SINGLE_PRIMARY_SIZE_CONDITION = new ParseField(MaxSinglePrimarySizeCondition.NAME); static { CONDITION_PARSER.declareString((conditions, s) -> - conditions.put(MaxAgeCondition.NAME, new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))), + conditions.put(MaxAgeCondition.NAME, + new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))), MAX_AGE_CONDITION); CONDITION_PARSER.declareLong((conditions, value) -> - conditions.put(MaxDocsCondition.NAME, new MaxDocsCondition(value)), MAX_DOCS_CONDITION); + conditions.put(MaxDocsCondition.NAME, + new MaxDocsCondition(value)), MAX_DOCS_CONDITION); CONDITION_PARSER.declareString((conditions, s) -> - conditions.put(MaxSizeCondition.NAME, new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))), + conditions.put(MaxSizeCondition.NAME, + new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))), MAX_SIZE_CONDITION); + CONDITION_PARSER.declareString((conditions, s) -> + conditions.put(MaxSinglePrimarySizeCondition.NAME, + new MaxSinglePrimarySizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSinglePrimarySizeCondition.NAME))), + MAX_SINGLE_PRIMARY_SIZE_CONDITION); PARSER.declareField((parser, request, context) -> CONDITION_PARSER.parse(parser, request.conditions, null), CONDITIONS, ObjectParser.ValueType.OBJECT); @@ -123,7 +132,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(rolloverTarget); out.writeOptionalString(newIndexName); out.writeBoolean(dryRun); - out.writeCollection(conditions.values(), StreamOutput::writeNamedWriteable); + out.writeCollection( + conditions.values().stream().filter(c -> c.includedInVersion(out.getVersion())).collect(Collectors.toList()), + StreamOutput::writeNamedWriteable); createIndexRequest.writeTo(out); } @@ -203,6 +214,16 @@ public void addMaxIndexSizeCondition(ByteSizeValue size) { this.conditions.put(maxSizeCondition.name, maxSizeCondition); } + /** + * Adds a size-based condition to check if the size of the largest primary shard is at least size. + */ + public void addMaxIndexSinglePrimarySizeCondition(ByteSizeValue size) { + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(size); + if (this.conditions.containsKey(maxSinglePrimarySizeCondition.name)) { + throw new IllegalArgumentException(maxSinglePrimarySizeCondition + " condition is already set"); + } + this.conditions.put(maxSinglePrimarySizeCondition.name, maxSinglePrimarySizeCondition); + } public boolean isDryRun() { return dryRun; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java index 173750da63b5a..2211055207e62 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java @@ -58,6 +58,11 @@ public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size){ return this; } + public RolloverRequestBuilder addMaxIndexSinglePrimarySizeCondition(ByteSizeValue size){ + this.request.addMaxIndexSinglePrimarySizeCondition(size); + return this; + } + public RolloverRequestBuilder dryRun(boolean dryRun) { this.request.dryRun(dryRun); return this; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index ee365e369f1c5..8a4a55158c67f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -23,9 +23,11 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardsObserver; import org.elasticsearch.action.support.IndicesOptions; @@ -47,6 +49,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -187,15 +190,27 @@ static Condition.Stats buildStats(@Nullable final IndexMetadata metadata, if (metadata == null) { return null; } else { - final DocsStats docsStats = Optional.ofNullable(statsResponse) - .map(stats -> stats.getIndex(metadata.getIndex().getName())) + final Optional indexStats = Optional.ofNullable(statsResponse) + .map(stats -> stats.getIndex(metadata.getIndex().getName())); + + final DocsStats docsStats = indexStats .map(stats -> stats.getPrimaries().getDocs()) .orElse(null); + final long maxSinglePrimarySize = indexStats.stream() + .map(IndexStats::getShards) + .filter(Objects::nonNull) + .flatMap(Arrays::stream) + .filter(shard -> shard.getShardRouting().primary()) + .map(ShardStats::getStats) + .mapToLong(shard -> shard.docs.getTotalSizeInBytes()) + .max().orElse(-1); + return new Condition.Stats( docsStats == null ? 0 : docsStats.getCount(), metadata.getCreationDate(), - new ByteSizeValue(docsStats == null ? 0 : docsStats.getTotalSizeInBytes()) + new ByteSizeValue(docsStats == null ? 0 : docsStats.getTotalSizeInBytes()), + new ByteSizeValue(maxSinglePrimarySize) ); } } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index dfa9914e74963..6d605d0e9fc4d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.resync.TransportResyncReplicationAction; import org.elasticsearch.common.ParseField; @@ -87,7 +88,9 @@ public static List getNamedWriteables() { return Arrays.asList( new NamedWriteableRegistry.Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new), new NamedWriteableRegistry.Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new), - new NamedWriteableRegistry.Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new)); + new NamedWriteableRegistry.Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new), + new NamedWriteableRegistry.Entry(Condition.class, MaxSinglePrimarySizeCondition.NAME, MaxSinglePrimarySizeCondition::new) + ); } public static List getNamedXContents() { @@ -97,7 +100,9 @@ public static List getNamedXContents() { new NamedXContentRegistry.Entry(Condition.class, new ParseField(MaxDocsCondition.NAME), (p, c) -> MaxDocsCondition.fromXContent(p)), new NamedXContentRegistry.Entry(Condition.class, new ParseField(MaxSizeCondition.NAME), (p, c) -> - MaxSizeCondition.fromXContent(p)) + MaxSizeCondition.fromXContent(p)), + new NamedXContentRegistry.Entry(Condition.class, new ParseField(MaxSinglePrimarySizeCondition.NAME), (p, c) -> + MaxSinglePrimarySizeCondition.fromXContent(p)) ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java index da51059e2d485..39f57dc1487cb 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.admin.indices.rollover; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -33,12 +32,12 @@ public void testMaxAge() { final MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(1)); long indexCreatedMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(61).getMillis(); - Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch, randomByteSize())); + Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch, randomByteSize(), randomByteSize())); assertThat(evaluate.condition, equalTo(maxAgeCondition)); assertThat(evaluate.matched, equalTo(true)); long indexCreatedNotMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(59).getMillis(); - evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch, randomByteSize())); + evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch, randomByteSize(), randomByteSize())); assertThat(evaluate.condition, equalTo(maxAgeCondition)); assertThat(evaluate.matched, equalTo(false)); } @@ -47,29 +46,46 @@ public void testMaxDocs() { final MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); long maxDocsMatch = randomIntBetween(100, 1000); - Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0, randomByteSize())); + Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0, randomByteSize(), randomByteSize())); assertThat(evaluate.condition, equalTo(maxDocsCondition)); assertThat(evaluate.matched, equalTo(true)); long maxDocsNotMatch = randomIntBetween(0, 99); - evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch, randomByteSize())); + evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch, randomByteSize(), randomByteSize())); assertThat(evaluate.condition, equalTo(maxDocsCondition)); assertThat(evaluate.matched, equalTo(false)); } public void testMaxSize() { - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 20), ByteSizeUnit.MB)); + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 20))); Condition.Result result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), - new ByteSizeValue(0, ByteSizeUnit.MB))); + ByteSizeValue.ofMb(0), randomByteSize())); assertThat(result.matched, equalTo(false)); result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), - new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB))); + ByteSizeValue.ofMb(randomIntBetween(0, 9)), randomByteSize())); assertThat(result.matched, equalTo(false)); result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), - new ByteSizeValue(randomIntBetween(20, 1000), ByteSizeUnit.MB))); + ByteSizeValue.ofMb(randomIntBetween(20, 1000)), randomByteSize())); + assertThat(result.matched, equalTo(true)); + } + + public void testMaxSinglePrimarySize() { + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + new MaxSinglePrimarySizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 20))); + + Condition.Result result = maxSinglePrimarySizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), + randomNonNegativeLong(), randomByteSize(), ByteSizeValue.ofMb(0))); + assertThat(result.matched, equalTo(false)); + + result = maxSinglePrimarySizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), + randomByteSize(), ByteSizeValue.ofMb(randomIntBetween(0, 9)))); + assertThat(result.matched, equalTo(false)); + + result = maxSinglePrimarySizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), + randomByteSize(), ByteSizeValue.ofMb(randomIntBetween(20, 1000)))); assertThat(result.matched, equalTo(true)); } @@ -85,9 +101,14 @@ public void testEqualsAndHashCode() { MaxSizeCondition maxSizeCondition = new MaxSizeCondition(randomByteSize()); EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxSizeCondition, condition -> new MaxSizeCondition(condition.value), condition -> new MaxSizeCondition(randomByteSize())); + + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(randomByteSize()); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxSinglePrimarySizeCondition, + condition -> new MaxSinglePrimarySizeCondition(condition.value), + condition -> new MaxSinglePrimarySizeCondition(randomByteSize())); } private static ByteSizeValue randomByteSize() { - return new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES); + return new ByteSizeValue(randomNonNegativeLong()); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 5678b91bcf7f9..b30a46b6b69a3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -68,17 +68,21 @@ public void testConditionsParsing() throws Exception { .field("max_age", "10d") .field("max_docs", 100) .field("max_size", "45gb") + .field("max_single_primary_size", "55gb") .endObject() .endObject(); request.fromXContent(createParser(builder)); Map> conditions = request.getConditions(); - assertThat(conditions.size(), equalTo(3)); + assertThat(conditions.size(), equalTo(4)); MaxAgeCondition maxAgeCondition = (MaxAgeCondition)conditions.get(MaxAgeCondition.NAME); assertThat(maxAgeCondition.value.getMillis(), equalTo(TimeValue.timeValueHours(24 * 10).getMillis())); MaxDocsCondition maxDocsCondition = (MaxDocsCondition)conditions.get(MaxDocsCondition.NAME); assertThat(maxDocsCondition.value, equalTo(100L)); MaxSizeCondition maxSizeCondition = (MaxSizeCondition)conditions.get(MaxSizeCondition.NAME); assertThat(maxSizeCondition.value.getBytes(), equalTo(ByteSizeUnit.GB.toBytes(45))); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + (MaxSinglePrimarySizeCondition)conditions.get(MaxSinglePrimarySizeCondition.NAME); + assertThat(maxSinglePrimarySizeCondition.value.getBytes(), equalTo(ByteSizeUnit.GB.toBytes(55))); } public void testParsingWithIndexSettings() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java index 7264b00f83ff3..678521a708683 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java @@ -54,8 +54,9 @@ private static Map randomResults(boolean allowNoItems) { private static final List>> conditionSuppliers = new ArrayList<>(); static { conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong()))); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 0b39ed8ef7e2c..5418cb259a696 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -46,10 +46,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.cache.request.RequestCacheStats; import org.elasticsearch.index.engine.SegmentsStats; @@ -101,7 +99,7 @@ public void testDocStatsSelectionFromPrimariesOnly() { final Condition condition = createTestCondition(); String indexName = randomAlphaOfLengthBetween(5, 7); - evaluateConditions(Sets.newHashSet(condition), + evaluateConditions(Set.of(condition), buildStats(createMetadata(indexName), createIndicesStatResponse(indexName, docsInShards, docsInPrimaryShards))); final ArgumentCaptor argument = ArgumentCaptor.forClass(Condition.Stats.class); verify(condition).evaluate(argument.capture()); @@ -112,30 +110,35 @@ public void testDocStatsSelectionFromPrimariesOnly() { public void testEvaluateConditions() { MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2)); MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB)); - final Set> conditions = Sets.newHashSet(maxAgeCondition, maxDocsCondition, maxSizeCondition); + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 100))); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + new MaxSinglePrimarySizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 100))); + final Set> conditions = Set.of(maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition); long matchMaxDocs = randomIntBetween(100, 1000); long notMatchMaxDocs = randomIntBetween(0, 99); - ByteSizeValue notMatchMaxSize = new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB); + ByteSizeValue notMatchMaxSize = ByteSizeValue.ofMb(randomIntBetween(0, 9)); long indexCreated = TimeValue.timeValueHours(3).getMillis(); - expectThrows(NullPointerException.class, () -> evaluateConditions(null, new Condition.Stats(0, 0, ByteSizeValue.ofMb(0)))); + expectThrows(NullPointerException.class, () -> evaluateConditions(null, + new Condition.Stats(0, 0, ByteSizeValue.ofMb(0), ByteSizeValue.ofMb(0)))); Map results = evaluateConditions(conditions, null); - assertThat(results.size(), equalTo(3)); + assertThat(results.size(), equalTo(4)); for (Boolean matched : results.values()) { assertThat(matched, equalTo(false)); } - results = evaluateConditions(conditions, new Condition.Stats(matchMaxDocs, indexCreated, ByteSizeValue.ofMb(120))); - assertThat(results.size(), equalTo(3)); + results = evaluateConditions(conditions, + new Condition.Stats(matchMaxDocs, indexCreated, ByteSizeValue.ofMb(120), ByteSizeValue.ofMb(120))); + assertThat(results.size(), equalTo(4)); for (Boolean matched : results.values()) { assertThat(matched, equalTo(true)); } - results = evaluateConditions(conditions, new Condition.Stats(notMatchMaxDocs, indexCreated, notMatchMaxSize)); - assertThat(results.size(), equalTo(3)); + results = evaluateConditions(conditions, + new Condition.Stats(notMatchMaxDocs, indexCreated, notMatchMaxSize, ByteSizeValue.ofMb(0))); + assertThat(results.size(), equalTo(4)); for (Map.Entry entry : results.entrySet()) { if (entry.getKey().equals(maxAgeCondition.toString())) { assertThat(entry.getValue(), equalTo(true)); @@ -143,6 +146,8 @@ public void testEvaluateConditions() { assertThat(entry.getValue(), equalTo(false)); } else if (entry.getKey().equals(maxSizeCondition.toString())) { assertThat(entry.getValue(), equalTo(false)); + } else if (entry.getKey().equals(maxSinglePrimarySizeCondition.toString())) { + assertThat(entry.getValue(), equalTo(false)); } else { fail("unknown condition result found " + entry.getKey()); } @@ -153,7 +158,9 @@ public void testEvaluateWithoutStats() { MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(randomIntBetween(1, 3))); MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomNonNegativeLong()); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())); - final Set> conditions = Sets.newHashSet(maxAgeCondition, maxDocsCondition, maxSizeCondition); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong())); + final Set> conditions = Set.of(maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition); final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) @@ -167,7 +174,7 @@ public void testEvaluateWithoutStats() { .settings(settings) .build(); Map results = evaluateConditions(conditions, buildStats(metadata, null)); - assertThat(results.size(), equalTo(3)); + assertThat(results.size(), equalTo(4)); for (Map.Entry entry : results.entrySet()) { if (entry.getKey().equals(maxAgeCondition.toString())) { @@ -176,6 +183,8 @@ public void testEvaluateWithoutStats() { assertThat(entry.getValue(), equalTo(false)); } else if (entry.getKey().equals(maxSizeCondition.toString())) { assertThat(entry.getValue(), equalTo(false)); + } else if (entry.getKey().equals(maxSinglePrimarySizeCondition.toString())) { + assertThat(entry.getValue(), equalTo(false)); } else { fail("unknown condition result found " + entry.getKey()); } @@ -185,8 +194,10 @@ public void testEvaluateWithoutStats() { public void testEvaluateWithoutMetadata() { MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2)); MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB)); - final Set> conditions = Sets.newHashSet(maxAgeCondition, maxDocsCondition, maxSizeCondition); + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 100))); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + new MaxSinglePrimarySizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 100))); + final Set> conditions = Set.of(maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition); final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) @@ -201,7 +212,7 @@ public void testEvaluateWithoutMetadata() { .build(); IndicesStatsResponse indicesStats = randomIndicesStatsResponse(new IndexMetadata[]{metadata}); Map results = evaluateConditions(conditions, buildStats(null, indicesStats)); - assertThat(results.size(), equalTo(3)); + assertThat(results.size(), equalTo(4)); results.forEach((k, v) -> assertFalse(v)); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index 73512aa35061c..aab2674b9d62c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.common.Strings; @@ -95,7 +96,8 @@ public void testIndexMetadataSerialization() throws IOException { List.of( new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), new MaxDocsCondition(randomNonNegativeLong()), - new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())) + new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())), + new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong())) ), randomNonNegativeLong())).build(); assertEquals(system, metadata.isSystem()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java index daddcba73a4eb..192ee042885ac 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.cli.MockTerminal; @@ -129,6 +130,7 @@ public void setup() throws IOException { Condition rolloverCondition = randomFrom( new MaxAgeCondition(new TimeValue(randomNonNegativeLong())), new MaxDocsCondition(randomNonNegativeLong()), + new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong())), new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())) ); From bdf3dc7a821dce110d2f0b05db75a972623cce98 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 20 Jan 2021 17:24:49 -0500 Subject: [PATCH 06/13] Client changes to keep up --- .../client/indices/rollover/RolloverRequest.java | 14 ++++++++++++++ .../indices/rollover/RolloverRequestTests.java | 11 ++++++++--- .../indices/rollover/RolloverResponseTests.java | 2 ++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java index 6383c0fd88d5f..c826cffb6edd7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.client.TimedRequest; import org.elasticsearch.client.indices.CreateIndexRequest; @@ -117,6 +118,19 @@ public RolloverRequest addMaxIndexSizeCondition(ByteSizeValue size) { this.conditions.put(maxSizeCondition.name(), maxSizeCondition); return this; } + + /** + * Adds a size-based condition to check if the size of the largest primary shard is at least size. + */ + public RolloverRequest addMaxIndexSinglePrimarySizeCondition(ByteSizeValue size) { + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(size); + if (this.conditions.containsKey(maxSinglePrimarySizeCondition.name())) { + throw new IllegalArgumentException(maxSinglePrimarySizeCondition + " condition is already set"); + } + this.conditions.put(maxSinglePrimarySizeCondition.name(), maxSinglePrimarySizeCondition); + return this; + } + /** * Returns all set conditions */ diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java index 57798c393db8f..9b5248f1e08cf 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -47,12 +48,16 @@ public void testConstructorAndFieldAssignments() { // test assignment of conditions MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(10)); - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000)); MaxDocsCondition maxDocsCondition = new MaxDocsCondition(10000L); - Condition[] expectedConditions = new Condition[] {maxAgeCondition, maxSizeCondition, maxDocsCondition}; + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000)); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(new ByteSizeValue(3000)); + Condition[] expectedConditions = new Condition[]{ + maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition + }; rolloverRequest.addMaxIndexAgeCondition(maxAgeCondition.value()); - rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value()); rolloverRequest.addMaxIndexDocsCondition(maxDocsCondition.value()); + rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value()); + rolloverRequest.addMaxIndexSinglePrimarySizeCondition(maxSinglePrimarySizeCondition.value()); List> requestConditions = new ArrayList<>(rolloverRequest.getConditions().values()); assertThat(requestConditions, containsInAnyOrder(expectedConditions)); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java index 582113873adfd..68f315471214a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -45,6 +46,7 @@ public class RolloverResponseTests extends ESTestCase { conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong()))); } public void testFromXContent() throws IOException { From af3f3e9aaedc8d6943bf6501e4d610ca4f33b843 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Tue, 26 Jan 2021 12:43:45 -0500 Subject: [PATCH 07/13] Whitespace --- .../action/admin/indices/rollover/RolloverRequestBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java index 2211055207e62..112d2ce2fe7df 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java @@ -53,12 +53,12 @@ public RolloverRequestBuilder addMaxIndexDocsCondition(long docs) { return this; } - public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size){ + public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size) { this.request.addMaxIndexSizeCondition(size); return this; } - public RolloverRequestBuilder addMaxIndexSinglePrimarySizeCondition(ByteSizeValue size){ + public RolloverRequestBuilder addMaxIndexSinglePrimarySizeCondition(ByteSizeValue size) { this.request.addMaxIndexSinglePrimarySizeCondition(size); return this; } From f6fe3f8156aa3ae9439fce627cdefaa51bd16822 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Tue, 26 Jan 2021 12:48:06 -0500 Subject: [PATCH 08/13] Default to 0 rather than -1 --- .../action/admin/indices/rollover/TransportRolloverAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index 8a4a55158c67f..8837c6146a611 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -204,7 +204,7 @@ static Condition.Stats buildStats(@Nullable final IndexMetadata metadata, .filter(shard -> shard.getShardRouting().primary()) .map(ShardStats::getStats) .mapToLong(shard -> shard.docs.getTotalSizeInBytes()) - .max().orElse(-1); + .max().orElse(0); return new Condition.Stats( docsStats == null ? 0 : docsStats.getCount(), From d00e2287e4b9de6ae5fb1b900847439b74ca0242 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 27 Jan 2021 15:41:33 -0500 Subject: [PATCH 09/13] s/MaxIndexSinglePrimarySize/MaxSinglePrimarySize/g --- .../client/indices/rollover/RolloverRequest.java | 2 +- .../client/indices/rollover/RolloverRequestTests.java | 2 +- .../action/admin/indices/rollover/RolloverRequest.java | 2 +- .../action/admin/indices/rollover/RolloverRequestBuilder.java | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java index c826cffb6edd7..c9ead5a9401fa 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java @@ -122,7 +122,7 @@ public RolloverRequest addMaxIndexSizeCondition(ByteSizeValue size) { /** * Adds a size-based condition to check if the size of the largest primary shard is at least size. */ - public RolloverRequest addMaxIndexSinglePrimarySizeCondition(ByteSizeValue size) { + public RolloverRequest addMaxSinglePrimarySizeCondition(ByteSizeValue size) { MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(size); if (this.conditions.containsKey(maxSinglePrimarySizeCondition.name())) { throw new IllegalArgumentException(maxSinglePrimarySizeCondition + " condition is already set"); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java index 9b5248f1e08cf..7d7de42d81a0b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java @@ -57,7 +57,7 @@ public void testConstructorAndFieldAssignments() { rolloverRequest.addMaxIndexAgeCondition(maxAgeCondition.value()); rolloverRequest.addMaxIndexDocsCondition(maxDocsCondition.value()); rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value()); - rolloverRequest.addMaxIndexSinglePrimarySizeCondition(maxSinglePrimarySizeCondition.value()); + rolloverRequest.addMaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition.value()); List> requestConditions = new ArrayList<>(rolloverRequest.getConditions().values()); assertThat(requestConditions, containsInAnyOrder(expectedConditions)); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 72cb594540bb3..db471466267b4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -217,7 +217,7 @@ public void addMaxIndexSizeCondition(ByteSizeValue size) { /** * Adds a size-based condition to check if the size of the largest primary shard is at least size. */ - public void addMaxIndexSinglePrimarySizeCondition(ByteSizeValue size) { + public void addMaxSinglePrimarySizeCondition(ByteSizeValue size) { MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(size); if (this.conditions.containsKey(maxSinglePrimarySizeCondition.name)) { throw new IllegalArgumentException(maxSinglePrimarySizeCondition + " condition is already set"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java index 112d2ce2fe7df..bcf2211f08973 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java @@ -58,8 +58,8 @@ public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size) { return this; } - public RolloverRequestBuilder addMaxIndexSinglePrimarySizeCondition(ByteSizeValue size) { - this.request.addMaxIndexSinglePrimarySizeCondition(size); + public RolloverRequestBuilder addMaxSinglePrimarySizeCondition(ByteSizeValue size) { + this.request.addMaxSinglePrimarySizeCondition(size); return this; } From cfce41bdcb0c0f97666a7a872e0c0e810a44b07b Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 29 Jan 2021 13:10:14 -0500 Subject: [PATCH 10/13] Docs, as well as docs and client tests --- .../elasticsearch/client/IndicesClientIT.java | 4 ++- .../IndicesClientDocumentationIT.java | 3 +- .../high-level/indices/rollover.asciidoc | 1 + .../reference/indices/rollover-index.asciidoc | 30 +++++++++++++++---- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 017f6f60cdfb7..280d444ed76e3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -969,15 +969,17 @@ public void testRollover() throws IOException { rolloverRequest.getCreateIndexRequest().mapping(mappings, XContentType.JSON); rolloverRequest.dryRun(false); rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); + rolloverRequest.addMaxSinglePrimarySizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertTrue(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); Map conditionStatus = rolloverResponse.getConditionStatus(); - assertEquals(3, conditionStatus.size()); + assertEquals(4, conditionStatus.size()); assertTrue(conditionStatus.get("[max_docs: 1]")); assertTrue(conditionStatus.get("[max_age: 1ms]")); assertFalse(conditionStatus.get("[max_size: 1mb]")); + assertFalse(conditionStatus.get("[max_single_primary_size: 1mb]")); assertEquals("test", rolloverResponse.getOldIndex()); assertEquals("test_new", rolloverResponse.getNewIndex()); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index d27071e0690a0..9bcee20f7ff9e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -1805,6 +1805,7 @@ public void testRolloverIndex() throws Exception { request.addMaxIndexAgeCondition(new TimeValue(7, TimeUnit.DAYS)); // <2> request.addMaxIndexDocsCondition(1000); // <3> request.addMaxIndexSizeCondition(new ByteSizeValue(5, ByteSizeUnit.GB)); // <4> + request.addMaxSinglePrimarySizeCondition(new ByteSizeValue(2, ByteSizeUnit.GB)); // <5> // end::rollover-index-request // tag::rollover-index-request-timeout @@ -1851,7 +1852,7 @@ public void testRolloverIndex() throws Exception { assertEquals("index-2", newIndex); assertFalse(isRolledOver); assertTrue(isDryRun); - assertEquals(3, conditionStatus.size()); + assertEquals(4, conditionStatus.size()); // tag::rollover-index-execute-listener ActionListener listener = new ActionListener() { diff --git a/docs/java-rest/high-level/indices/rollover.asciidoc b/docs/java-rest/high-level/indices/rollover.asciidoc index 6b7a82a11ae2b..e1e3298544b2b 100644 --- a/docs/java-rest/high-level/indices/rollover.asciidoc +++ b/docs/java-rest/high-level/indices/rollover.asciidoc @@ -24,6 +24,7 @@ The new index argument is optional, and can be set to null <2> Condition on the age of the index <3> Condition on the number of documents in the index <4> Condition on the size of the index +<5> Condition on the size of the largest primary shard of the index ==== Optional arguments The following arguments can optionally be provided: diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index 9810d6dd53154..97a9f246b506d 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -18,7 +18,8 @@ POST /alias1/_rollover/my-index-000002 "conditions": { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } ---- @@ -158,6 +159,16 @@ Replicas are not counted toward the maximum index size. TIP: To see the current index size, use the <> API. The `pri.store.size` value shows the combined size of all primary shards. + +`max_single_primary_size`:: +(Optional, <>) +Maximum primary shard size. +This is the maximum size of the primary shards in the index. As with `max_size`, +replicas are ignored. + +TIP: To see the current shard size, use the <> API. +The `store` value shows the size each shard, and `prirep` indicates whether a +shard is a primary (`p`) or a replica (`r`). -- include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings] @@ -187,7 +198,8 @@ POST /logs_write/_rollover <2> "conditions": { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } -------------------------------------------------- @@ -213,6 +225,7 @@ The API returns the following response: "[max_age: 7d]": false, "[max_docs: 1000]": true, "[max_size: 5gb]": false, + "[max_single_primary_size: 2gb]": false } } -------------------------------------------------- @@ -245,7 +258,8 @@ POST /my-data-stream/_rollover <2> "conditions" : { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } -------------------------------------------------- @@ -279,6 +293,7 @@ The API returns the following response: "[max_age: 7d]": false, "[max_docs: 1000]": true, "[max_size: 5gb]": false, + "[max_single_primary_size: 2gb]": false } } -------------------------------------------------- @@ -324,7 +339,8 @@ POST /logs_write/_rollover "conditions" : { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" }, "settings": { "index.number_of_shards": 2 @@ -354,7 +370,8 @@ POST /my_alias/_rollover/my_new_index_name "conditions": { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } -------------------------------------------------- @@ -451,7 +468,8 @@ POST /logs_write/_rollover?dry_run "conditions" : { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } -------------------------------------------------- From 20a7b3e237ceb2443d67118f0d364b62a17bcee5 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 29 Jan 2021 13:34:58 -0500 Subject: [PATCH 11/13] Add yml test Just a copy/paste + search&replace on 30_max_size_condition.yml, nothing special. --- .../35_max_single_primary_size_condition.yml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml new file mode 100644 index 0000000000000..4be7ac0ce0fe4 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml @@ -0,0 +1,55 @@ +--- +"Rollover with max_single_primary_size condition": + + # create index with alias and replica + - do: + indices.create: + index: logs-1 + wait_for_active_shards: 1 + body: + aliases: + logs_search: {} + + # index a document + - do: + index: + index: logs-1 + id: "1" + body: { "foo": "hello world" } + refresh: true + + # perform alias rollover with a large max_single_primary_size, no action. + - do: + indices.rollover: + alias: "logs_search" + wait_for_active_shards: 1 + body: + conditions: + max_single_primary_size: 100mb + + - match: { conditions: { "[max_single_primary_size: 100mb]": false } } + - match: { rolled_over: false } + + # perform alias rollover with a small max_single_primary_size, got action. + - do: + indices.rollover: + alias: "logs_search" + wait_for_active_shards: 1 + body: + conditions: + max_single_primary_size: 10b + + - match: { conditions: { "[max_single_primary_size: 10b]": true } } + - match: { rolled_over: true } + + # perform alias rollover on an empty index, no action. + - do: + indices.rollover: + alias: "logs_search" + wait_for_active_shards: 1 + body: + conditions: + max_single_primary_size: 1b + + - match: { conditions: { "[max_single_primary_size: 1b]": false } } + - match: { rolled_over: false } From 0f4bc10a9ebefeeee36f5fd2be6996d171786055 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Tue, 2 Feb 2021 12:35:50 -0500 Subject: [PATCH 12/13] Add "version:" restriction to the yml test for bwc --- .../indices.rollover/35_max_single_primary_size_condition.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml index 4be7ac0ce0fe4..cd95894adb5b7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml @@ -1,5 +1,8 @@ --- "Rollover with max_single_primary_size condition": + - skip: + version: " - 7.99.99" + reason: max_single_primary_size condition was introduced in 8.0.0 # create index with alias and replica - do: From 52927b79dd3c80a59173614e8a1208a9e8481b97 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Tue, 2 Feb 2021 13:13:21 -0500 Subject: [PATCH 13/13] Add testRolloverMaxSinglePrimarySize It's just a copy/paste/edit of testRolloverMaxSize, but it's still got some value. --- .../admin/indices/rollover/RolloverIT.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java index 7b31cbead5c3e..ba310ae6c0197 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java @@ -445,6 +445,62 @@ public void testRolloverMaxSize() throws Exception { } } + public void testRolloverMaxSinglePrimarySize() throws Exception { + assertAcked(prepareCreate("test-1").addAlias(new Alias("test_alias")).get()); + int numDocs = randomIntBetween(10, 20); + for (int i = 0; i < numDocs; i++) { + indexDoc("test-1", Integer.toString(i), "field", "foo-" + i); + } + flush("test-1"); + refresh("test_alias"); + + // A large max_single_primary_size + { + final RolloverResponse response = client().admin().indices() + .prepareRolloverIndex("test_alias") + .addMaxSinglePrimarySizeCondition(new ByteSizeValue(randomIntBetween(100, 50 * 1024), ByteSizeUnit.MB)) + .get(); + assertThat(response.getOldIndex(), equalTo("test-1")); + assertThat(response.getNewIndex(), equalTo("test-000002")); + assertThat("No rollover with a large max_single_primary_size condition", response.isRolledOver(), equalTo(false)); + final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-1"); + assertThat(oldIndex.getRolloverInfos().size(), equalTo(0)); + } + + // A small max_single_primary_size + { + ByteSizeValue maxSinglePrimarySizeCondition = new ByteSizeValue(randomIntBetween(1, 20), ByteSizeUnit.BYTES); + long beforeTime = client().threadPool().absoluteTimeInMillis() - 1000L; + final RolloverResponse response = client().admin().indices() + .prepareRolloverIndex("test_alias") + .addMaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition) + .get(); + assertThat(response.getOldIndex(), equalTo("test-1")); + assertThat(response.getNewIndex(), equalTo("test-000002")); + assertThat("Should rollover with a small max_single_primary_size condition", response.isRolledOver(), equalTo(true)); + final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-1"); + List> metConditions = oldIndex.getRolloverInfos().get("test_alias").getMetConditions(); + assertThat(metConditions.size(), equalTo(1)); + assertThat(metConditions.get(0).toString(), + equalTo(new MaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition).toString())); + assertThat(oldIndex.getRolloverInfos().get("test_alias").getTime(), + is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L)))); + } + + // An empty index + { + final RolloverResponse response = client().admin().indices() + .prepareRolloverIndex("test_alias") + .addMaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)) + .get(); + assertThat(response.getOldIndex(), equalTo("test-000002")); + assertThat(response.getNewIndex(), equalTo("test-000003")); + assertThat("No rollover with an empty index", response.isRolledOver(), equalTo(false)); + final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-000002"); + assertThat(oldIndex.getRolloverInfos().size(), equalTo(0)); + } + } + public void testRejectIfAliasFoundInTemplate() throws Exception { client().admin().indices().preparePutTemplate("logs") .setPatterns(Collections.singletonList("logs-*")).addAlias(new Alias("logs-write")).get();