Add size-based condition to the index rollover API (#27160)

This is to add a max_size condition to the index rollover API. We use
a totalSizeInBytes from DocsStats to evaluate this condition.

Closes #27004

Author: dnhatn

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java

@@ -19,8 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 
@@ -38,6 +40,9 @@ public abstract class Condition<T> implements NamedWriteable {
             new ParseField(MaxAgeCondition.NAME));
         PARSER.declareLong((conditions, value) ->
             conditions.add(new MaxDocsCondition(value)), new ParseField(MaxDocsCondition.NAME));
+        PARSER.declareString((conditions, s) ->
+                conditions.add(new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))),
+            new ParseField(MaxSizeCondition.NAME));
     }
 
     protected T value;
@@ -49,6 +54,14 @@ protected Condition(String name) {
 
     public abstract Result evaluate(Stats stats);
 
+    /**
+     * Checks if this condition is available in a specific version.
+     * This makes sure BWC when introducing a new condition which is not recognized by older versions.
+     */
+    boolean includedInVersion(Version version) {
+        return true;
+    }
+
     @Override
     public final String toString() {
         return "[" + name + ": " + value + "]";
@@ -60,10 +73,12 @@ public final String toString() {
     public static class Stats {
         public final long numDocs;
         public final long indexCreated;
+        public final ByteSizeValue indexSize;
 
-        public Stats(long numDocs, long indexCreated) {
+        public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) {
             this.numDocs = numDocs;
             this.indexCreated = indexCreated;
+            this.indexSize = indexSize;
         }
     }
 

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+
+import java.io.IOException;
+
+/**
+ * A size-based condition for an index size.
+ * Evaluates to <code>true</code> if the index size is at least {@link #value}.
+ */
+public class MaxSizeCondition extends Condition<ByteSizeValue> {
+    public static final String NAME = "max_size";
+
+    public MaxSizeCondition(ByteSizeValue value) {
+        super(NAME);
+        this.value = value;
+    }
+
+    public MaxSizeCondition(StreamInput in) throws IOException {
+        super(NAME);
+        this.value = new ByteSizeValue(in.readVLong(), ByteSizeUnit.BYTES);
+    }
+
+    @Override
+    public Result evaluate(Stats stats) {
+        return new Result(this, stats.indexSize.getBytes() >= value.getBytes());
+    }
+
+    @Override
+    boolean includedInVersion(Version version) {
+        return version.onOrAfter(Version.V_7_0_0_alpha1);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(value.getBytes());
+    }
+}

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 
@@ -106,7 +107,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(dryRun);
         out.writeVInt(conditions.size());
         for (Condition condition : conditions) {
-            out.writeNamedWriteable(condition);
+            if (condition.includedInVersion(out.getVersion())) {
+                out.writeNamedWriteable(condition);
+            }
         }
         createIndexRequest.writeTo(out);
     }
@@ -155,6 +158,13 @@ public void addMaxIndexDocsCondition(long numDocs) {
         this.conditions.add(new MaxDocsCondition(numDocs));
     }
 
+    /**
+     * Adds a size-based condition to check if the index size is at least <code>size</code>.
+     */
+    public void addMaxIndexSizeCondition(ByteSizeValue size) {
+        this.conditions.add(new MaxSizeCondition(size));
+    }
+
     /**
      * Sets rollover index creation request to override index settings when
      * the rolled over index has to be created

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 
 
@@ -52,6 +53,11 @@ public RolloverRequestBuilder addMaxIndexDocsCondition(long docs) {
         return this;
     }
 
+    public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size){
+        this.request.addMaxIndexSizeCondition(size);
+        return this;
+    }
+
     public RolloverRequestBuilder dryRun(boolean dryRun) {
         this.request.dryRun(dryRun);
         return this;

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java

@@ -43,6 +43,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -195,7 +196,8 @@ static String generateRolloverIndexName(String sourceIndexName, IndexNameExpress
     static Set<Condition.Result> evaluateConditions(final Set<Condition> conditions,
                                                     final DocsStats docsStats, final IndexMetaData metaData) {
         final long numDocs = docsStats == null ? 0 : docsStats.getCount();
-        final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate());
+        final long indexSize = docsStats == null ? 0 : docsStats.getTotalSizeInBytes();
+        final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate(), new ByteSizeValue(indexSize));
         return conditions.stream()
             .map(condition -> condition.evaluate(stats))
             .collect(Collectors.toSet());

core/src/main/java/org/elasticsearch/indices/IndicesModule.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.action.admin.indices.rollover.Condition;
 import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
 import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
+import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
 import org.elasticsearch.action.resync.TransportResyncReplicationAction;
 import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
 import org.elasticsearch.common.geo.ShapesAvailability;
@@ -79,6 +80,7 @@ public IndicesModule(List<MapperPlugin> mapperPlugins) {
     private void registerBuiltinWritables() {
         namedWritables.add(new Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new));
         namedWritables.add(new Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new));
+        namedWritables.add(new Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new));
     }
 
     public List<Entry> getNamedWriteables() {

core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 
@@ -30,12 +32,12 @@ public void testMaxAge() throws Exception {
         final MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(1));
 
         long indexCreatedMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(61).getMillis();
-        Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch));
+        Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch, randomByteSize()));
         assertThat(evaluate.condition, equalTo(maxAgeCondition));
         assertThat(evaluate.matched, equalTo(true));
 
         long indexCreatedNotMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(59).getMillis();
-        evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch));
+        evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch, randomByteSize()));
         assertThat(evaluate.condition, equalTo(maxAgeCondition));
         assertThat(evaluate.matched, equalTo(false));
     }
@@ -44,13 +46,33 @@ public void testMaxDocs() throws Exception {
         final MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
 
         long maxDocsMatch = randomIntBetween(100, 1000);
-        Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0));
+        Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0, randomByteSize()));
         assertThat(evaluate.condition, equalTo(maxDocsCondition));
         assertThat(evaluate.matched, equalTo(true));
 
         long maxDocsNotMatch = randomIntBetween(0, 99);
-        evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch));
+        evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch, randomByteSize()));
         assertThat(evaluate.condition, equalTo(maxDocsCondition));
         assertThat(evaluate.matched, equalTo(false));
     }
+
+    public void testMaxSize() throws Exception {
+        MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 20), ByteSizeUnit.MB));
+
+        Condition.Result result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
+            new ByteSizeValue(0, ByteSizeUnit.MB)));
+        assertThat(result.matched, equalTo(false));
+
+        result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
+            new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB)));
+        assertThat(result.matched, equalTo(false));
+
+        result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
+            new ByteSizeValue(randomIntBetween(20, 1000), ByteSizeUnit.MB)));
+        assertThat(result.matched, equalTo(true));
+    }
+
+    private ByteSizeValue randomByteSize() {
+        return new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES);
+    }
 }

core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java

@@ -19,13 +19,15 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -36,9 +38,15 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.hasProperty;
+import static org.hamcrest.Matchers.is;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class RolloverIT extends ESIntegTestCase {
@@ -128,15 +136,23 @@ public void testRolloverConditionsNotMet() throws Exception {
         index("test_index-0", "type1", "1", "field", "value");
         flush("test_index-0");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
+            .addMaxIndexSizeCondition(new ByteSizeValue(10, ByteSizeUnit.MB))
             .addMaxIndexAgeCondition(TimeValue.timeValueHours(4)).get();
         assertThat(response.getOldIndex(), equalTo("test_index-0"));
         assertThat(response.getNewIndex(), equalTo("test_index-000001"));
         assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(false));
-        assertThat(response.getConditionStatus().size(), equalTo(1));
-        final Map.Entry<String, Boolean> conditionEntry = response.getConditionStatus().iterator().next();
-        assertThat(conditionEntry.getKey(), equalTo(new MaxAgeCondition(TimeValue.timeValueHours(4)).toString()));
-        assertThat(conditionEntry.getValue(), equalTo(false));
+        assertThat(response.getConditionStatus().size(), equalTo(2));
+
+
+        assertThat(response.getConditionStatus(), everyItem(hasProperty("value", is(false))));
+        Set<String> conditions = response.getConditionStatus().stream()
+            .map(Map.Entry::getKey)
+            .collect(Collectors.toSet());
+        assertThat(conditions, containsInAnyOrder(
+            new MaxSizeCondition(new ByteSizeValue(10, ByteSizeUnit.MB)).toString(),
+            new MaxAgeCondition(TimeValue.timeValueHours(4)).toString()));
+
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-0");
         assertTrue(oldIndex.getAliases().containsKey("test_alias"));
@@ -218,4 +234,47 @@ public void testRolloverWithDateMath() {
         assertThat(response.isRolledOver(), equalTo(true));
         assertThat(response.getConditionStatus().size(), equalTo(0));
     }
+
+    public void testRolloverMaxSize() throws Exception {
+        assertAcked(prepareCreate("test-1").addAlias(new Alias("test_alias")).get());
+        int numDocs = randomIntBetween(10, 20);
+        for (int i = 0; i < numDocs; i++) {
+            index("test-1", "doc", Integer.toString(i), "field", "foo-" + i);
+        }
+        flush("test-1");
+        refresh("test_alias");
+
+        // A large max_size
+        {
+            final RolloverResponse response = client().admin().indices()
+                .prepareRolloverIndex("test_alias")
+                .addMaxIndexSizeCondition(new ByteSizeValue(randomIntBetween(100, 50 * 1024), ByteSizeUnit.MB))
+                .get();
+            assertThat(response.getOldIndex(), equalTo("test-1"));
+            assertThat(response.getNewIndex(), equalTo("test-000002"));
+            assertThat("No rollover with a large max_size condition", response.isRolledOver(), equalTo(false));
+        }
+
+        // A small max_size
+        {
+            final RolloverResponse response = client().admin().indices()
+                .prepareRolloverIndex("test_alias")
+                .addMaxIndexSizeCondition(new ByteSizeValue(randomIntBetween(1, 20), ByteSizeUnit.BYTES))
+                .get();
+            assertThat(response.getOldIndex(), equalTo("test-1"));
+            assertThat(response.getNewIndex(), equalTo("test-000002"));
+            assertThat("Should rollover with a small max_size condition", response.isRolledOver(), equalTo(true));
+        }
+
+        // An empty index
+        {
+            final RolloverResponse response = client().admin().indices()
+                .prepareRolloverIndex("test_alias")
+                .addMaxIndexSizeCondition(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES))
+                .get();
+            assertThat(response.getOldIndex(), equalTo("test-000002"));
+            assertThat(response.getNewIndex(), equalTo("test-000003"));
+            assertThat("No rollover with an empty index", response.isRolledOver(), equalTo(false));
+        }
+    }
 }