diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java index 678a0c9e921d3..1202ee84e9a53 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.client.TimedRequest; import org.elasticsearch.client.indices.CreateIndexRequest; @@ -94,6 +95,7 @@ public RolloverRequest addMaxIndexDocsCondition(long numDocs) { this.conditions.put(maxDocsCondition.name(), maxDocsCondition); return this; } + /** * Adds a size-based condition to check if the index size is at least size. */ @@ -105,6 +107,19 @@ public RolloverRequest addMaxIndexSizeCondition(ByteSizeValue size) { this.conditions.put(maxSizeCondition.name(), maxSizeCondition); return this; } + + /** + * Adds a size-based condition to check if the size of the largest primary shard is at least size. + */ + public RolloverRequest addMaxSinglePrimarySizeCondition(ByteSizeValue size) { + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(size); + if (this.conditions.containsKey(maxSinglePrimarySizeCondition.name())) { + throw new IllegalArgumentException(maxSinglePrimarySizeCondition + " condition is already set"); + } + this.conditions.put(maxSinglePrimarySizeCondition.name(), maxSinglePrimarySizeCondition); + return this; + } + /** * Returns all set conditions */ diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 9afb025665f9f..4bdd03c7e92bb 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -1213,15 +1213,17 @@ public void testRollover() throws IOException { rolloverRequest.getCreateIndexRequest().mapping(mappings, XContentType.JSON); rolloverRequest.dryRun(false); rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); + rolloverRequest.addMaxSinglePrimarySizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertTrue(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); Map conditionStatus = rolloverResponse.getConditionStatus(); - assertEquals(3, conditionStatus.size()); + assertEquals(4, conditionStatus.size()); assertTrue(conditionStatus.get("[max_docs: 1]")); assertTrue(conditionStatus.get("[max_age: 1ms]")); assertFalse(conditionStatus.get("[max_size: 1mb]")); + assertFalse(conditionStatus.get("[max_single_primary_size: 1mb]")); assertEquals("test", rolloverResponse.getOldIndex()); assertEquals("test_new", rolloverResponse.getNewIndex()); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index c3ed7dbf73f38..4a9073ecb1dd7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -1899,6 +1899,7 @@ public void testRolloverIndex() throws Exception { request.addMaxIndexAgeCondition(new TimeValue(7, TimeUnit.DAYS)); // <2> request.addMaxIndexDocsCondition(1000); // <3> request.addMaxIndexSizeCondition(new ByteSizeValue(5, ByteSizeUnit.GB)); // <4> + request.addMaxSinglePrimarySizeCondition(new ByteSizeValue(2, ByteSizeUnit.GB)); // <5> // end::rollover-index-request // tag::rollover-index-request-timeout @@ -1945,7 +1946,7 @@ public void testRolloverIndex() throws Exception { assertEquals("index-2", newIndex); assertFalse(isRolledOver); assertTrue(isDryRun); - assertEquals(3, conditionStatus.size()); + assertEquals(4, conditionStatus.size()); // tag::rollover-index-execute-listener ActionListener listener = new ActionListener() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java index 4211ea034039d..e8b4d3d20644d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -36,12 +37,16 @@ public void testConstructorAndFieldAssignments() { // test assignment of conditions MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(10)); - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000)); MaxDocsCondition maxDocsCondition = new MaxDocsCondition(10000L); - Condition[] expectedConditions = new Condition[] {maxAgeCondition, maxSizeCondition, maxDocsCondition}; + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000)); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(new ByteSizeValue(3000)); + Condition[] expectedConditions = new Condition[]{ + maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition + }; rolloverRequest.addMaxIndexAgeCondition(maxAgeCondition.value()); - rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value()); rolloverRequest.addMaxIndexDocsCondition(maxDocsCondition.value()); + rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value()); + rolloverRequest.addMaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition.value()); List> requestConditions = new ArrayList<>(rolloverRequest.getConditions().values()); assertThat(requestConditions, containsInAnyOrder(expectedConditions)); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java index 9dfb21d0ee227..b583b16cf246c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -36,8 +37,9 @@ public class RolloverResponseTests extends ESTestCase { private static final List>> conditionSuppliers = new ArrayList<>(); static { conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong()))); } public void testFromXContent() throws IOException { diff --git a/docs/java-rest/high-level/indices/rollover.asciidoc b/docs/java-rest/high-level/indices/rollover.asciidoc index 6b7a82a11ae2b..e1e3298544b2b 100644 --- a/docs/java-rest/high-level/indices/rollover.asciidoc +++ b/docs/java-rest/high-level/indices/rollover.asciidoc @@ -24,6 +24,7 @@ The new index argument is optional, and can be set to null <2> Condition on the age of the index <3> Condition on the number of documents in the index <4> Condition on the size of the index +<5> Condition on the size of the largest primary shard of the index ==== Optional arguments The following arguments can optionally be provided: diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index 6ea727ec337b7..eb855c096ad4b 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -18,7 +18,8 @@ POST /alias1/_rollover/my-index-000002 "conditions": { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } ---- @@ -165,6 +166,16 @@ Replicas are not counted toward the maximum index size. TIP: To see the current index size, use the <> API. The `pri.store.size` value shows the combined size of all primary shards. + +`max_single_primary_size`:: +(Optional, <>) +Maximum primary shard size. +This is the maximum size of the primary shards in the index. As with `max_size`, +replicas are ignored. + +TIP: To see the current shard size, use the <> API. +The `store` value shows the size each shard, and `prirep` indicates whether a +shard is a primary (`p`) or a replica (`r`). -- include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings] @@ -194,7 +205,8 @@ POST /logs_write/_rollover <2> "conditions": { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } -------------------------------------------------- @@ -220,6 +232,7 @@ The API returns the following response: "[max_age: 7d]": false, "[max_docs: 1000]": true, "[max_size: 5gb]": false, + "[max_single_primary_size: 2gb]": false } } -------------------------------------------------- @@ -252,7 +265,8 @@ POST /my-data-stream/_rollover <2> "conditions" : { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } -------------------------------------------------- @@ -286,6 +300,7 @@ The API returns the following response: "[max_age: 7d]": false, "[max_docs: 1000]": true, "[max_size: 5gb]": false, + "[max_single_primary_size: 2gb]": false } } -------------------------------------------------- @@ -332,7 +347,8 @@ POST /logs_write/_rollover "conditions" : { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" }, "settings": { "index.number_of_shards": 2 @@ -359,7 +375,8 @@ POST /my_alias/_rollover/my_new_index_name "conditions": { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } -------------------------------------------------- @@ -457,7 +474,8 @@ POST /logs_write/_rollover?dry_run "conditions" : { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb" + "max_size": "5gb", + "max_single_primary_size": "2gb" } } -------------------------------------------------- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml new file mode 100644 index 0000000000000..cfdcd54969b6b --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml @@ -0,0 +1,58 @@ +--- +"Rollover with max_single_primary_size condition": + - skip: + version: " - 7.11.99" + reason: max_single_primary_size condition was introduced in 7.12.0 + + # create index with alias and replica + - do: + indices.create: + index: logs-1 + wait_for_active_shards: 1 + body: + aliases: + logs_search: {} + + # index a document + - do: + index: + index: logs-1 + id: "1" + body: { "foo": "hello world" } + refresh: true + + # perform alias rollover with a large max_single_primary_size, no action. + - do: + indices.rollover: + alias: "logs_search" + wait_for_active_shards: 1 + body: + conditions: + max_single_primary_size: 100mb + + - match: { conditions: { "[max_single_primary_size: 100mb]": false } } + - match: { rolled_over: false } + + # perform alias rollover with a small max_single_primary_size, got action. + - do: + indices.rollover: + alias: "logs_search" + wait_for_active_shards: 1 + body: + conditions: + max_single_primary_size: 10b + + - match: { conditions: { "[max_single_primary_size: 10b]": true } } + - match: { rolled_over: true } + + # perform alias rollover on an empty index, no action. + - do: + indices.rollover: + alias: "logs_search" + wait_for_active_shards: 1 + body: + conditions: + max_single_primary_size: 1b + + - match: { conditions: { "[max_single_primary_size: 1b]": false } } + - match: { rolled_over: false } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java index d92f2e93d39cb..a75ca549658b4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java @@ -435,6 +435,62 @@ public void testRolloverMaxSize() throws Exception { } } + public void testRolloverMaxSinglePrimarySize() throws Exception { + assertAcked(prepareCreate("test-1").addAlias(new Alias("test_alias")).get()); + int numDocs = randomIntBetween(10, 20); + for (int i = 0; i < numDocs; i++) { + index("test-1", "doc", Integer.toString(i), "field", "foo-" + i); + } + flush("test-1"); + refresh("test_alias"); + + // A large max_single_primary_size + { + final RolloverResponse response = client().admin().indices() + .prepareRolloverIndex("test_alias") + .addMaxSinglePrimarySizeCondition(new ByteSizeValue(randomIntBetween(100, 50 * 1024), ByteSizeUnit.MB)) + .get(); + assertThat(response.getOldIndex(), equalTo("test-1")); + assertThat(response.getNewIndex(), equalTo("test-000002")); + assertThat("No rollover with a large max_single_primary_size condition", response.isRolledOver(), equalTo(false)); + final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-1"); + assertThat(oldIndex.getRolloverInfos().size(), equalTo(0)); + } + + // A small max_single_primary_size + { + ByteSizeValue maxSinglePrimarySizeCondition = new ByteSizeValue(randomIntBetween(1, 20), ByteSizeUnit.BYTES); + long beforeTime = client().threadPool().absoluteTimeInMillis() - 1000L; + final RolloverResponse response = client().admin().indices() + .prepareRolloverIndex("test_alias") + .addMaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition) + .get(); + assertThat(response.getOldIndex(), equalTo("test-1")); + assertThat(response.getNewIndex(), equalTo("test-000002")); + assertThat("Should rollover with a small max_single_primary_size condition", response.isRolledOver(), equalTo(true)); + final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-1"); + List> metConditions = oldIndex.getRolloverInfos().get("test_alias").getMetConditions(); + assertThat(metConditions.size(), equalTo(1)); + assertThat(metConditions.get(0).toString(), + equalTo(new MaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition).toString())); + assertThat(oldIndex.getRolloverInfos().get("test_alias").getTime(), + is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L)))); + } + + // An empty index + { + final RolloverResponse response = client().admin().indices() + .prepareRolloverIndex("test_alias") + .addMaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)) + .get(); + assertThat(response.getOldIndex(), equalTo("test-000002")); + assertThat(response.getNewIndex(), equalTo("test-000003")); + assertThat("No rollover with an empty index", response.isRolledOver(), equalTo(false)); + final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-000002"); + assertThat(oldIndex.getRolloverInfos().size(), equalTo(0)); + } + } + public void testRejectIfAliasFoundInTemplate() throws Exception { client().admin().indices().preparePutTemplate("logs") .setPatterns(Collections.singletonList("logs-*")).addAlias(new Alias("logs-write")).get(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java index 410a131f5885a..363a377f394ba 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java @@ -75,11 +75,13 @@ public static class Stats { public final long numDocs; public final long indexCreated; public final ByteSizeValue indexSize; + public final ByteSizeValue maxSinglePrimarySize; - public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) { + public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize, ByteSizeValue maxSinglePrimarySize) { this.numDocs = numDocs; this.indexCreated = indexCreated; this.indexSize = indexSize; + this.maxSinglePrimarySize = maxSinglePrimarySize; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSinglePrimarySizeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSinglePrimarySizeCondition.java new file mode 100644 index 0000000000000..2821dcb47b007 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSinglePrimarySizeCondition.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.indices.rollover; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +/** + * A size-based condition for the primary shards within an index. + * Evaluates to true if the size of the largest primary shard is at least {@link #value}. + */ +public class MaxSinglePrimarySizeCondition extends Condition { + public static final String NAME = "max_single_primary_size"; + + public MaxSinglePrimarySizeCondition(ByteSizeValue value) { + super(NAME); + this.value = value; + } + + public MaxSinglePrimarySizeCondition(StreamInput in) throws IOException { + super(NAME); + this.value = new ByteSizeValue(in.readVLong(), ByteSizeUnit.BYTES); + } + + @Override + public Result evaluate(Stats stats) { + return new Result(this, stats.maxSinglePrimarySize.getBytes() >= value.getBytes()); + } + + @Override + public String getWriteableName() { return NAME; } + + @Override + public void writeTo(StreamOutput out) throws IOException { + // While we technically could serialize this with value.writeTo(...), that would + // require doing the song and dance around backwards compatibility for this value. Since + // in this case the deserialized version is not displayed to a user, it's okay to simply use + // bytes. + out.writeVLong(value.getBytes()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.field(NAME, value.getStringRep()); + } + + public static MaxSinglePrimarySizeCondition fromXContent(XContentParser parser) throws IOException { + if (parser.nextToken() == XContentParser.Token.VALUE_STRING) { + return new MaxSinglePrimarySizeCondition(ByteSizeValue.parseBytesSizeValue(parser.text(), NAME)); + } else { + throw new IllegalArgumentException("invalid token: " + parser.currentToken()); + } + } + + @Override + boolean includedInVersion(Version version) { + return version.onOrAfter(Version.V_7_12_0); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 0e0bd4be2bc2f..21e2f162a8d6f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -45,16 +46,24 @@ public class RolloverRequest extends AcknowledgedRequest implem private static final ParseField MAX_AGE_CONDITION = new ParseField(MaxAgeCondition.NAME); private static final ParseField MAX_DOCS_CONDITION = new ParseField(MaxDocsCondition.NAME); private static final ParseField MAX_SIZE_CONDITION = new ParseField(MaxSizeCondition.NAME); + private static final ParseField MAX_SINGLE_PRIMARY_SIZE_CONDITION = new ParseField(MaxSinglePrimarySizeCondition.NAME); static { CONDITION_PARSER.declareString((conditions, s) -> - conditions.put(MaxAgeCondition.NAME, new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))), - MAX_AGE_CONDITION); + conditions.put(MaxAgeCondition.NAME, + new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))), + MAX_AGE_CONDITION); CONDITION_PARSER.declareLong((conditions, value) -> - conditions.put(MaxDocsCondition.NAME, new MaxDocsCondition(value)), MAX_DOCS_CONDITION); + conditions.put(MaxDocsCondition.NAME, + new MaxDocsCondition(value)), MAX_DOCS_CONDITION); CONDITION_PARSER.declareString((conditions, s) -> - conditions.put(MaxSizeCondition.NAME, new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))), - MAX_SIZE_CONDITION); + conditions.put(MaxSizeCondition.NAME, + new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))), + MAX_SIZE_CONDITION); + CONDITION_PARSER.declareString((conditions, s) -> + conditions.put(MaxSinglePrimarySizeCondition.NAME, + new MaxSinglePrimarySizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSinglePrimarySizeCondition.NAME))), + MAX_SINGLE_PRIMARY_SIZE_CONDITION); PARSER.declareField((parser, request, context) -> CONDITION_PARSER.parse(parser, request.conditions, null), CONDITIONS, ObjectParser.ValueType.OBJECT); @@ -121,12 +130,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(rolloverTarget); out.writeOptionalString(newIndexName); out.writeBoolean(dryRun); - out.writeVInt(conditions.size()); - for (Condition condition : conditions.values()) { - if (condition.includedInVersion(out.getVersion())) { - out.writeNamedWriteable(condition); - } - } + out.writeCollection( + conditions.values().stream().filter(c -> c.includedInVersion(out.getVersion())).collect(Collectors.toList()), + StreamOutput::writeNamedWriteable); createIndexRequest.writeTo(out); } @@ -158,6 +164,7 @@ public void setRolloverTarget(String rolloverTarget) { public void setNewIndexName(String newIndexName) { this.newIndexName = newIndexName; } + /** * Sets if the rollover should not be executed when conditions are met */ @@ -205,6 +212,16 @@ public void addMaxIndexSizeCondition(ByteSizeValue size) { this.conditions.put(maxSizeCondition.name, maxSizeCondition); } + /** + * Adds a size-based condition to check if the size of the largest primary shard is at least size. + */ + public void addMaxSinglePrimarySizeCondition(ByteSizeValue size) { + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(size); + if (this.conditions.containsKey(maxSinglePrimarySizeCondition.name)) { + throw new IllegalArgumentException(maxSinglePrimarySizeCondition + " condition is already set"); + } + this.conditions.put(maxSinglePrimarySizeCondition.name, maxSinglePrimarySizeCondition); + } public boolean isDryRun() { return dryRun; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java index ad878b428612c..03ee4820a0d58 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java @@ -43,11 +43,16 @@ public RolloverRequestBuilder addMaxIndexDocsCondition(long docs) { return this; } - public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size){ + public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size) { this.request.addMaxIndexSizeCondition(size); return this; } + public RolloverRequestBuilder addMaxSinglePrimarySizeCondition(ByteSizeValue size) { + this.request.addMaxSinglePrimarySizeCondition(size); + return this; + } + public RolloverRequestBuilder dryRun(boolean dryRun) { this.request.dryRun(dryRun); return this; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index bbabb2a2fecb2..10fb1739dd09c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -12,9 +12,11 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardsObserver; import org.elasticsearch.action.support.IndicesOptions; @@ -36,11 +38,14 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Main class to swap the index pointed to by an alias, given some conditions @@ -94,6 +99,7 @@ protected void masterOperation(Task task, final RolloverRequest rolloverRequest, // synchronization (in this case, the submitStateUpdateTask which is serialized on the master node), where we then regenerate the // names and re-check conditions. More explanation follows inline below. client.execute(IndicesStatsAction.INSTANCE, statsRequest, + ActionListener.wrap(statsResponse -> { // Now that we have the stats for the cluster, we need to know the // names of the index for which we should evaluate @@ -108,7 +114,7 @@ protected void masterOperation(Task task, final RolloverRequest rolloverRequest, // Evaluate the conditions, so that we can tell without a cluster state update whether a rollover would occur. final Map trialConditionResults = evaluateConditions(rolloverRequest.getConditions().values(), - metadata.index(trialSourceIndexName), statsResponse); + buildStats(metadata.index(trialSourceIndexName), statsResponse)); // If this is a dry run, return with the results without invoking a cluster state update if (rolloverRequest.isDryRun()) { @@ -145,7 +151,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { // Re-evaluate the conditions, now with our final source index name final Map postConditionResults = evaluateConditions(rolloverRequest.getConditions().values(), - metadata.index(sourceIndexName), statsResponse); + buildStats(metadata.index(sourceIndexName), statsResponse)); final List> metConditions = rolloverRequest.getConditions().values().stream() .filter(condition -> postConditionResults.get(condition.toString())).collect(Collectors.toList()); // Update the final condition results so they can be used when returning the response @@ -214,30 +220,56 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } static Map evaluateConditions(final Collection> conditions, - @Nullable final DocsStats docsStats, - @Nullable final IndexMetadata metadata) { - if (metadata == null) { - return conditions.stream().collect(Collectors.toMap(Condition::toString, cond -> false)); + @Nullable final Condition.Stats stats) { + Objects.requireNonNull(conditions, "conditions must not be null"); + + if (stats != null) { + return conditions.stream() + .map(condition -> condition.evaluate(stats)) + .collect(Collectors.toMap(result -> result.condition.toString(), result -> result.matched)); + } else { + // no conditions matched + return conditions.stream() + .collect(Collectors.toMap(Condition::toString, cond -> false)); } - final long numDocs = docsStats == null ? 0 : docsStats.getCount(); - final long indexSize = docsStats == null ? 0 : docsStats.getTotalSizeInBytes(); - final Condition.Stats stats = new Condition.Stats(numDocs, metadata.getCreationDate(), new ByteSizeValue(indexSize)); - return conditions.stream() - .map(condition -> condition.evaluate(stats)) - .collect(Collectors.toMap(result -> result.condition.toString(), result -> result.matched)); } - static Map evaluateConditions(final Collection> conditions, - @Nullable final IndexMetadata metadata, - @Nullable final IndicesStatsResponse statsResponse) { + static Condition.Stats buildStats(@Nullable final IndexMetadata metadata, + @Nullable final IndicesStatsResponse statsResponse) { if (metadata == null) { - return conditions.stream().collect(Collectors.toMap(Condition::toString, cond -> false)); + return null; } else { - final DocsStats docsStats = Optional.ofNullable(statsResponse) - .map(stats -> stats.getIndex(metadata.getIndex().getName())) - .map(indexStats -> indexStats.getPrimaries().getDocs()) + final Optional indexStats = Optional.ofNullable(statsResponse) + .map(stats -> stats.getIndex(metadata.getIndex().getName())); + + final DocsStats docsStats = indexStats + .map(stats -> stats.getPrimaries().getDocs()) .orElse(null); - return evaluateConditions(conditions, docsStats, metadata); + + final long maxSinglePrimarySize = optionalStream(indexStats) + .map(IndexStats::getShards) + .filter(Objects::nonNull) + .flatMap(Arrays::stream) + .filter(shard -> shard.getShardRouting().primary()) + .map(ShardStats::getStats) + .mapToLong(shard -> shard.docs.getTotalSizeInBytes()) + .max().orElse(0); + + return new Condition.Stats( + docsStats == null ? 0 : docsStats.getCount(), + metadata.getCreationDate(), + new ByteSizeValue(docsStats == null ? 0 : docsStats.getTotalSizeInBytes()), + new ByteSizeValue(maxSinglePrimarySize) + ); + } + } + + // helper because java 8 doesn't have Optional.stream + private static Stream optionalStream(Optional o) { + if (o.isPresent()) { + return Stream.of(o.get()); + } else { + return Stream.empty(); } } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java b/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java index 507527e5ee4cf..99cf04b8f284e 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java @@ -73,14 +73,6 @@ public long getTotalSizeInBytes() { return totalSizeInBytes; } - /** - * Returns the average size in bytes of all documents in this stats. - */ - public long getAverageSizeInBytes() { - long totalDocs = count + deleted; - return totalDocs == 0 ? 0 : totalSizeInBytes / totalDocs; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(count); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index d7ff2286b27c1..ac4feefd26aa8 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.resync.TransportResyncReplicationAction; import org.elasticsearch.common.ParseField; @@ -83,6 +84,8 @@ private void registerBuiltinWritables() { namedWritables.add(new NamedWriteableRegistry.Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new)); namedWritables.add(new NamedWriteableRegistry.Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new)); namedWritables.add(new NamedWriteableRegistry.Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new)); + namedWritables.add(new NamedWriteableRegistry.Entry(Condition.class, + MaxSinglePrimarySizeCondition.NAME, MaxSinglePrimarySizeCondition::new)); } public List getNamedWriteables() { @@ -96,7 +99,9 @@ public static List getNamedXContents() { new NamedXContentRegistry.Entry(Condition.class, new ParseField(MaxDocsCondition.NAME), (p, c) -> MaxDocsCondition.fromXContent(p)), new NamedXContentRegistry.Entry(Condition.class, new ParseField(MaxSizeCondition.NAME), (p, c) -> - MaxSizeCondition.fromXContent(p)) + MaxSizeCondition.fromXContent(p)), + new NamedXContentRegistry.Entry(Condition.class, new ParseField(MaxSinglePrimarySizeCondition.NAME), (p, c) -> + MaxSinglePrimarySizeCondition.fromXContent(p)) ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java index 24b72da00184c..0b843c022d087 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.action.admin.indices.rollover; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -22,12 +21,12 @@ public void testMaxAge() { final MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(1)); long indexCreatedMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(61).getMillis(); - Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch, randomByteSize())); + Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch, randomByteSize(), randomByteSize())); assertThat(evaluate.condition, equalTo(maxAgeCondition)); assertThat(evaluate.matched, equalTo(true)); long indexCreatedNotMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(59).getMillis(); - evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch, randomByteSize())); + evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch, randomByteSize(), randomByteSize())); assertThat(evaluate.condition, equalTo(maxAgeCondition)); assertThat(evaluate.matched, equalTo(false)); } @@ -36,47 +35,69 @@ public void testMaxDocs() { final MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); long maxDocsMatch = randomIntBetween(100, 1000); - Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0, randomByteSize())); + Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0, randomByteSize(), randomByteSize())); assertThat(evaluate.condition, equalTo(maxDocsCondition)); assertThat(evaluate.matched, equalTo(true)); long maxDocsNotMatch = randomIntBetween(0, 99); - evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch, randomByteSize())); + evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch, randomByteSize(), randomByteSize())); assertThat(evaluate.condition, equalTo(maxDocsCondition)); assertThat(evaluate.matched, equalTo(false)); } public void testMaxSize() { - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 20), ByteSizeUnit.MB)); + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 20))); Condition.Result result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), - new ByteSizeValue(0, ByteSizeUnit.MB))); + ByteSizeValue.ofMb(0), randomByteSize())); assertThat(result.matched, equalTo(false)); result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), - new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB))); + ByteSizeValue.ofMb(randomIntBetween(0, 9)), randomByteSize())); assertThat(result.matched, equalTo(false)); result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), - new ByteSizeValue(randomIntBetween(20, 1000), ByteSizeUnit.MB))); + ByteSizeValue.ofMb(randomIntBetween(20, 1000)), randomByteSize())); + assertThat(result.matched, equalTo(true)); + } + + public void testMaxSinglePrimarySize() { + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + new MaxSinglePrimarySizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 20))); + + Condition.Result result = maxSinglePrimarySizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), + randomNonNegativeLong(), randomByteSize(), ByteSizeValue.ofMb(0))); + assertThat(result.matched, equalTo(false)); + + result = maxSinglePrimarySizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), + randomByteSize(), ByteSizeValue.ofMb(randomIntBetween(0, 9)))); + assertThat(result.matched, equalTo(false)); + + result = maxSinglePrimarySizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(), + randomByteSize(), ByteSizeValue.ofMb(randomIntBetween(20, 1000)))); assertThat(result.matched, equalTo(true)); } public void testEqualsAndHashCode() { + MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(randomNonNegativeLong())); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxAgeCondition, condition -> new MaxAgeCondition(condition.value), + condition -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomLong()); EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxDocsCondition, condition -> new MaxDocsCondition(condition.value), - condition -> new MaxDocsCondition(randomLong())); + condition -> new MaxDocsCondition(randomLong())); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(randomByteSize()); EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxSizeCondition, condition -> new MaxSizeCondition(condition.value), - condition -> new MaxSizeCondition(randomByteSize())); + condition -> new MaxSizeCondition(randomByteSize())); - MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(randomNonNegativeLong())); - EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxAgeCondition, condition -> new MaxAgeCondition(condition.value), - condition -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(randomByteSize()); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(maxSinglePrimarySizeCondition, + condition -> new MaxSinglePrimarySizeCondition(condition.value), + condition -> new MaxSinglePrimarySizeCondition(randomByteSize())); } private static ByteSizeValue randomByteSize() { - return new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES); + return new ByteSizeValue(randomNonNegativeLong()); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index f0f23b315e7e3..285aae03ec91f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -61,17 +61,21 @@ public void testConditionsParsing() throws Exception { .field("max_age", "10d") .field("max_docs", 100) .field("max_size", "45gb") + .field("max_single_primary_size", "55gb") .endObject() .endObject(); request.fromXContent(false, createParser(builder)); Map> conditions = request.getConditions(); - assertThat(conditions.size(), equalTo(3)); + assertThat(conditions.size(), equalTo(4)); MaxAgeCondition maxAgeCondition = (MaxAgeCondition)conditions.get(MaxAgeCondition.NAME); assertThat(maxAgeCondition.value.getMillis(), equalTo(TimeValue.timeValueHours(24 * 10).getMillis())); MaxDocsCondition maxDocsCondition = (MaxDocsCondition)conditions.get(MaxDocsCondition.NAME); assertThat(maxDocsCondition.value, equalTo(100L)); MaxSizeCondition maxSizeCondition = (MaxSizeCondition)conditions.get(MaxSizeCondition.NAME); assertThat(maxSizeCondition.value.getBytes(), equalTo(ByteSizeUnit.GB.toBytes(45))); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + (MaxSinglePrimarySizeCondition)conditions.get(MaxSinglePrimarySizeCondition.NAME); + assertThat(maxSinglePrimarySizeCondition.value.getBytes(), equalTo(ByteSizeUnit.GB.toBytes(55))); } public void testParsingWithIndexSettings() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java index 630b639120b11..8273a0eb7c59a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java @@ -48,8 +48,9 @@ private static Map randomResults(boolean allowNoItems) { private static final List>> conditionSuppliers = new ArrayList<>(); static { conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong()))); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 462af8c89d3e2..8cba8c158cda9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -35,10 +35,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.cache.request.RequestCacheStats; import org.elasticsearch.index.engine.SegmentsStats; @@ -69,6 +67,7 @@ import java.util.Set; import static java.util.Collections.emptyList; +import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.buildStats; import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.evaluateConditions; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -89,8 +88,8 @@ public void testDocStatsSelectionFromPrimariesOnly() { final Condition condition = createTestCondition(); String indexName = randomAlphaOfLengthBetween(5, 7); - evaluateConditions(Sets.newHashSet(condition), createMetadata(indexName), - createIndicesStatResponse(indexName, docsInShards, docsInPrimaryShards)); + evaluateConditions(org.elasticsearch.common.collect.Set.of(condition), + buildStats(createMetadata(indexName), createIndicesStatResponse(indexName, docsInShards, docsInPrimaryShards))); final ArgumentCaptor argument = ArgumentCaptor.forClass(Condition.Stats.class); verify(condition).evaluate(argument.capture()); @@ -98,33 +97,38 @@ public void testDocStatsSelectionFromPrimariesOnly() { } public void testEvaluateConditions() { - MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2)); - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB)); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 100))); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + new MaxSinglePrimarySizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 100))); + final Set> conditions = org.elasticsearch.common.collect.Set.of( + maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition); long matchMaxDocs = randomIntBetween(100, 1000); long notMatchMaxDocs = randomIntBetween(0, 99); - ByteSizeValue notMatchMaxSize = new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB); - final Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - final IndexMetadata metadata = IndexMetadata.builder(randomAlphaOfLength(10)) - .creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(3).getMillis()) - .settings(settings) - .build(); - final Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); - Map results = evaluateConditions(conditions, - new DocsStats(matchMaxDocs, 0L, ByteSizeUnit.MB.toBytes(120)), metadata); - assertThat(results.size(), equalTo(3)); + ByteSizeValue notMatchMaxSize = ByteSizeValue.ofMb(randomIntBetween(0, 9)); + long indexCreated = TimeValue.timeValueHours(3).getMillis(); + + expectThrows(NullPointerException.class, () -> evaluateConditions(null, + new Condition.Stats(0, 0, ByteSizeValue.ofMb(0), ByteSizeValue.ofMb(0)))); + + Map results = evaluateConditions(conditions, null); + assertThat(results.size(), equalTo(4)); + for (Boolean matched : results.values()) { + assertThat(matched, equalTo(false)); + } + + results = evaluateConditions(conditions, + new Condition.Stats(matchMaxDocs, indexCreated, ByteSizeValue.ofMb(120), ByteSizeValue.ofMb(120))); + assertThat(results.size(), equalTo(4)); for (Boolean matched : results.values()) { assertThat(matched, equalTo(true)); } - results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0, notMatchMaxSize.getBytes()), metadata); - assertThat(results.size(), equalTo(3)); + results = evaluateConditions(conditions, + new Condition.Stats(notMatchMaxDocs, indexCreated, notMatchMaxSize, ByteSizeValue.ofMb(0))); + assertThat(results.size(), equalTo(4)); for (Map.Entry entry : results.entrySet()) { if (entry.getKey().equals(maxAgeCondition.toString())) { assertThat(entry.getValue(), equalTo(true)); @@ -132,18 +136,23 @@ public void testEvaluateConditions() { assertThat(entry.getValue(), equalTo(false)); } else if (entry.getKey().equals(maxSizeCondition.toString())) { assertThat(entry.getValue(), equalTo(false)); + } else if (entry.getKey().equals(maxSinglePrimarySizeCondition.toString())) { + assertThat(entry.getValue(), equalTo(false)); } else { fail("unknown condition result found " + entry.getKey()); } } } - public void testEvaluateWithoutDocStats() { - MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomNonNegativeLong()); + public void testEvaluateWithoutStats() { MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(randomIntBetween(1, 3))); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomNonNegativeLong()); MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong())); + final Set> conditions = org.elasticsearch.common.collect.Set.of( + maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition); - Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) @@ -155,8 +164,8 @@ public void testEvaluateWithoutDocStats() { .creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(randomIntBetween(5, 10)).getMillis()) .settings(settings) .build(); - Map results = evaluateConditions(conditions, null, metadata); - assertThat(results.size(), equalTo(3)); + Map results = evaluateConditions(conditions, buildStats(metadata, null)); + assertThat(results.size(), equalTo(4)); for (Map.Entry entry : results.entrySet()) { if (entry.getKey().equals(maxAgeCondition.toString())) { @@ -165,6 +174,8 @@ public void testEvaluateWithoutDocStats() { assertThat(entry.getValue(), equalTo(false)); } else if (entry.getKey().equals(maxSizeCondition.toString())) { assertThat(entry.getValue(), equalTo(false)); + } else if (entry.getKey().equals(maxSinglePrimarySizeCondition.toString())) { + assertThat(entry.getValue(), equalTo(false)); } else { fail("unknown condition result found " + entry.getKey()); } @@ -172,16 +183,13 @@ public void testEvaluateWithoutDocStats() { } public void testEvaluateWithoutMetadata() { - MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2)); - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB)); - - long matchMaxDocs = randomIntBetween(100, 1000); - final Set> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition); - Map results = evaluateConditions(conditions, - new DocsStats(matchMaxDocs, 0L, ByteSizeUnit.MB.toBytes(120)), null); - assertThat(results.size(), equalTo(3)); - results.forEach((k, v) -> assertFalse(v)); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 100))); + MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = + new MaxSinglePrimarySizeCondition(ByteSizeValue.ofMb(randomIntBetween(10, 100))); + final Set> conditions = org.elasticsearch.common.collect.Set.of( + maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition); final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) @@ -195,12 +203,11 @@ public void testEvaluateWithoutMetadata() { .settings(settings) .build(); IndicesStatsResponse indicesStats = randomIndicesStatsResponse(new IndexMetadata[]{metadata}); - Map results2 = evaluateConditions(conditions, null, indicesStats); - assertThat(results2.size(), equalTo(3)); - results2.forEach((k, v) -> assertFalse(v)); + Map results = evaluateConditions(conditions, buildStats(null, indicesStats)); + assertThat(results.size(), equalTo(4)); + results.forEach((k, v) -> assertFalse(v)); } - public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPrimariesFromWriteIndex() throws Exception { final TransportService mockTransportService = mock(TransportService.class); final ClusterService mockClusterService = mock(ClusterService.class); @@ -234,20 +241,20 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr assert statsResponse.getTotal().getDocs().getCount() == (total + total); final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("logs-index-000001") - .putAlias(AliasMetadata.builder("logs-alias").writeIndex(false).build()).settings(settings(Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(1); + .putAlias(AliasMetadata.builder("logs-alias").writeIndex(false).build()).settings(settings(Version.CURRENT)) + .numberOfShards(1).numberOfReplicas(1); final IndexMetadata.Builder indexMetadata2 = IndexMetadata.builder("logs-index-000002") - .putAlias(AliasMetadata.builder("logs-alias").writeIndex(true).build()).settings(settings(Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(1); + .putAlias(AliasMetadata.builder("logs-alias").writeIndex(true).build()).settings(settings(Version.CURRENT)) + .numberOfShards(1).numberOfReplicas(1); final ClusterState stateBefore = ClusterState.builder(ClusterName.DEFAULT) - .metadata(Metadata.builder().put(indexMetadata).put(indexMetadata2)).build(); + .metadata(Metadata.builder().put(indexMetadata).put(indexMetadata2)).build(); when(mockCreateIndexService.applyCreateIndexRequest(any(), any(), anyBoolean())).thenReturn(stateBefore); when(mdIndexAliasesService.applyAliasActions(any(), any())).thenReturn(stateBefore); MetadataRolloverService rolloverService = new MetadataRolloverService(mockThreadPool, mockCreateIndexService, mdIndexAliasesService, mockIndexNameExpressionResolver); final TransportRolloverAction transportRolloverAction = new TransportRolloverAction(mockTransportService, mockClusterService, - mockThreadPool, mockActionFilters, mockIndexNameExpressionResolver, rolloverService, mockClient); + mockThreadPool, mockActionFilters, mockIndexNameExpressionResolver, rolloverService, mockClient); // For given alias, verify that condition evaluation fails when the condition doc count is greater than the primaries doc count // (primaries from only write index is considered) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index b13a72f4c499a..aaa4477311345 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.common.Strings; @@ -35,7 +36,6 @@ import org.junit.Before; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -85,9 +85,12 @@ public void testIndexMetadataSerialization() throws IOException { .putCustom("my_custom", customMap) .putRolloverInfo( new RolloverInfo(randomAlphaOfLength(5), - Arrays.asList(new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), + org.elasticsearch.common.collect.List.of( + new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), + new MaxDocsCondition(randomNonNegativeLong()), new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())), - new MaxDocsCondition(randomNonNegativeLong())), + new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong())) + ), randomNonNegativeLong())).build(); assertEquals(system, metadata.isSystem()); diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java index 8ac3d3c70788e..9bd6f46e3e2c6 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java @@ -141,7 +141,6 @@ public void testNoOpEngineStats() throws Exception { assertEquals(expectedDocStats.getCount(), noOpEngine.docStats().getCount()); assertEquals(expectedDocStats.getDeleted(), noOpEngine.docStats().getDeleted()); assertEquals(expectedDocStats.getTotalSizeInBytes(), noOpEngine.docStats().getTotalSizeInBytes()); - assertEquals(expectedDocStats.getAverageSizeInBytes(), noOpEngine.docStats().getAverageSizeInBytes()); assertEquals(expectedSegmentStats.getCount(), noOpEngine.segmentsStats(includeFileSize, true).getCount()); // don't compare memory in bytes since we load the index with term-dict off-heap assertEquals(expectedSegmentStats.getFileSizes().size(), diff --git a/server/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java b/server/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java index 61001f4037cfc..f1339859879f7 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/DocsStatsTests.java @@ -17,36 +17,17 @@ public class DocsStatsTests extends ESTestCase { - public void testCalculateAverageDocSize() throws Exception { - DocsStats stats = new DocsStats(10, 2, 120); - assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); - - stats.add(new DocsStats(0, 0, 0)); - assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); - - stats.add(new DocsStats(8, 30, 480)); - assertThat(stats.getCount(), equalTo(18L)); - assertThat(stats.getDeleted(), equalTo(32L)); - assertThat(stats.getTotalSizeInBytes(), equalTo(600L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(12L)); - } - public void testUninitialisedShards() { DocsStats stats = new DocsStats(0, 0, -1); assertThat(stats.getTotalSizeInBytes(), equalTo(-1L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(0L)); stats.add(new DocsStats(0, 0, -1)); assertThat(stats.getTotalSizeInBytes(), equalTo(-1L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(0L)); stats.add(new DocsStats(1, 0, 10)); assertThat(stats.getTotalSizeInBytes(), equalTo(10L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); stats.add(new DocsStats(0, 0, -1)); assertThat(stats.getTotalSizeInBytes(), equalTo(10L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(10L)); stats.add(new DocsStats(1, 0, 20)); assertThat(stats.getTotalSizeInBytes(), equalTo(30L)); - assertThat(stats.getAverageSizeInBytes(), equalTo(15L)); } public void testSerialize() throws Exception { @@ -58,7 +39,7 @@ public void testSerialize() throws Exception { DocsStats cloneStats = new DocsStats(in); assertThat(cloneStats.getCount(), equalTo(originalStats.getCount())); assertThat(cloneStats.getDeleted(), equalTo(originalStats.getDeleted())); - assertThat(cloneStats.getAverageSizeInBytes(), equalTo(originalStats.getAverageSizeInBytes())); + assertThat(cloneStats.getTotalSizeInBytes(), equalTo(originalStats.getTotalSizeInBytes())); } } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 522ecdf4d0cac..ebe5808710b2e 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2913,7 +2913,7 @@ public void testDocStats() throws Exception { assertTrue(searcher.getIndexReader().numDocs() <= docsStats.getCount()); } assertThat(docsStats.getDeleted(), equalTo(0L)); - assertThat(docsStats.getAverageSizeInBytes(), greaterThan(0L)); + assertThat(docsStats.getTotalSizeInBytes(), greaterThan(0L)); } final List ids = randomSubsetOf( @@ -2975,7 +2975,7 @@ public void testDocStats() throws Exception { final DocsStats docStats = indexShard.docStats(); assertThat(docStats.getCount(), equalTo(numDocs)); assertThat(docStats.getDeleted(), equalTo(0L)); - assertThat(docStats.getAverageSizeInBytes(), greaterThan(0L)); + assertThat(docStats.getTotalSizeInBytes(), greaterThan(0L)); } } finally { closeShards(indexShard);