Skip to content

Commit 5b899da

Browse files
joegallojoegallo
authored
Add max_single_primary_size as a condition for the rollover index API (#67842) (#68489)
1 parent 59f9f41 commit 5b899da

File tree

24 files changed

+443
-144
lines changed

24 files changed

+443
-144
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.elasticsearch.action.admin.indices.rollover.Condition;
1111
import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
1212
import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
13+
import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition;
1314
import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
1415
import org.elasticsearch.client.TimedRequest;
1516
import org.elasticsearch.client.indices.CreateIndexRequest;
@@ -94,6 +95,7 @@ public RolloverRequest addMaxIndexDocsCondition(long numDocs) {
9495
this.conditions.put(maxDocsCondition.name(), maxDocsCondition);
9596
return this;
9697
}
98+
9799
/**
98100
* Adds a size-based condition to check if the index size is at least <code>size</code>.
99101
*/
@@ -105,6 +107,19 @@ public RolloverRequest addMaxIndexSizeCondition(ByteSizeValue size) {
105107
this.conditions.put(maxSizeCondition.name(), maxSizeCondition);
106108
return this;
107109
}
110+
111+
/**
112+
* Adds a size-based condition to check if the size of the largest primary shard is at least <code>size</code>.
113+
*/
114+
public RolloverRequest addMaxSinglePrimarySizeCondition(ByteSizeValue size) {
115+
MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(size);
116+
if (this.conditions.containsKey(maxSinglePrimarySizeCondition.name())) {
117+
throw new IllegalArgumentException(maxSinglePrimarySizeCondition + " condition is already set");
118+
}
119+
this.conditions.put(maxSinglePrimarySizeCondition.name(), maxSinglePrimarySizeCondition);
120+
return this;
121+
}
122+
108123
/**
109124
* Returns all set conditions
110125
*/

‎client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1213,15 +1213,17 @@ public void testRollover() throws IOException {
12131213
rolloverRequest.getCreateIndexRequest().mapping(mappings, XContentType.JSON);
12141214
rolloverRequest.dryRun(false);
12151215
rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB));
1216+
rolloverRequest.addMaxSinglePrimarySizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB));
12161217
RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover,
12171218
highLevelClient().indices()::rolloverAsync);
12181219
assertTrue(rolloverResponse.isRolledOver());
12191220
assertFalse(rolloverResponse.isDryRun());
12201221
Map<String, Boolean> conditionStatus = rolloverResponse.getConditionStatus();
1221-
assertEquals(3, conditionStatus.size());
1222+
assertEquals(4, conditionStatus.size());
12221223
assertTrue(conditionStatus.get("[max_docs: 1]"));
12231224
assertTrue(conditionStatus.get("[max_age: 1ms]"));
12241225
assertFalse(conditionStatus.get("[max_size: 1mb]"));
1226+
assertFalse(conditionStatus.get("[max_single_primary_size: 1mb]"));
12251227
assertEquals("test", rolloverResponse.getOldIndex());
12261228
assertEquals("test_new", rolloverResponse.getNewIndex());
12271229
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1899,6 +1899,7 @@ public void testRolloverIndex() throws Exception {
18991899
request.addMaxIndexAgeCondition(new TimeValue(7, TimeUnit.DAYS)); // <2>
19001900
request.addMaxIndexDocsCondition(1000); // <3>
19011901
request.addMaxIndexSizeCondition(new ByteSizeValue(5, ByteSizeUnit.GB)); // <4>
1902+
request.addMaxSinglePrimarySizeCondition(new ByteSizeValue(2, ByteSizeUnit.GB)); // <5>
19021903
// end::rollover-index-request
19031904

19041905
// tag::rollover-index-request-timeout
@@ -1945,7 +1946,7 @@ public void testRolloverIndex() throws Exception {
19451946
assertEquals("index-2", newIndex);
19461947
assertFalse(isRolledOver);
19471948
assertTrue(isDryRun);
1948-
assertEquals(3, conditionStatus.size());
1949+
assertEquals(4, conditionStatus.size());
19491950

19501951
// tag::rollover-index-execute-listener
19511952
ActionListener<RolloverResponse> listener = new ActionListener<RolloverResponse>() {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.action.admin.indices.rollover.Condition;
1212
import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
1313
import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
14+
import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition;
1415
import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
1516
import org.elasticsearch.common.unit.ByteSizeValue;
1617
import org.elasticsearch.common.unit.TimeValue;
@@ -36,12 +37,16 @@ public void testConstructorAndFieldAssignments() {
3637

3738
// test assignment of conditions
3839
MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(10));
39-
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000));
4040
MaxDocsCondition maxDocsCondition = new MaxDocsCondition(10000L);
41-
Condition<?>[] expectedConditions = new Condition<?>[] {maxAgeCondition, maxSizeCondition, maxDocsCondition};
41+
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000));
42+
MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(new ByteSizeValue(3000));
43+
Condition<?>[] expectedConditions = new Condition<?>[]{
44+
maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition
45+
};
4246
rolloverRequest.addMaxIndexAgeCondition(maxAgeCondition.value());
43-
rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value());
4447
rolloverRequest.addMaxIndexDocsCondition(maxDocsCondition.value());
48+
rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value());
49+
rolloverRequest.addMaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition.value());
4550
List<Condition<?>> requestConditions = new ArrayList<>(rolloverRequest.getConditions().values());
4651
assertThat(requestConditions, containsInAnyOrder(expectedConditions));
4752
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.action.admin.indices.rollover.Condition;
1212
import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
1313
import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
14+
import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition;
1415
import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
1516
import org.elasticsearch.common.unit.ByteSizeValue;
1617
import org.elasticsearch.common.unit.TimeValue;
@@ -36,8 +37,9 @@ public class RolloverResponseTests extends ESTestCase {
3637
private static final List<Supplier<Condition<?>>> conditionSuppliers = new ArrayList<>();
3738
static {
3839
conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong())));
39-
conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())));
4040
conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong()));
41+
conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())));
42+
conditionSuppliers.add(() -> new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong())));
4143
}
4244

4345
public void testFromXContent() throws IOException {

‎docs/java-rest/high-level/indices/rollover.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ The new index argument is optional, and can be set to null
2424
<2> Condition on the age of the index
2525
<3> Condition on the number of documents in the index
2626
<4> Condition on the size of the index
27+
<5> Condition on the size of the largest primary shard of the index
2728

2829
==== Optional arguments
2930
The following arguments can optionally be provided:

‎docs/reference/indices/rollover-index.asciidoc‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ POST /alias1/_rollover/my-index-000002
1818
"conditions": {
1919
"max_age": "7d",
2020
"max_docs": 1000,
21-
"max_size": "5gb"
21+
"max_size": "5gb",
22+
"max_single_primary_size": "2gb"
2223
}
2324
}
2425
----
@@ -165,6 +166,16 @@ Replicas are not counted toward the maximum index size.
165166

166167
TIP: To see the current index size, use the <<cat-indices, _cat indices>> API.
167168
The `pri.store.size` value shows the combined size of all primary shards.
169+
170+
`max_single_primary_size`::
171+
(Optional, <<byte-units, byte units>>)
172+
Maximum primary shard size.
173+
This is the maximum size of the primary shards in the index. As with `max_size`,
174+
replicas are ignored.
175+
176+
TIP: To see the current shard size, use the <<cat-shards, _cat shards>> API.
177+
The `store` value shows the size each shard, and `prirep` indicates whether a
178+
shard is a primary (`p`) or a replica (`r`).
168179
--
169180

170181
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings]
@@ -194,7 +205,8 @@ POST /logs_write/_rollover <2>
194205
"conditions": {
195206
"max_age": "7d",
196207
"max_docs": 1000,
197-
"max_size": "5gb"
208+
"max_size": "5gb",
209+
"max_single_primary_size": "2gb"
198210
}
199211
}
200212
--------------------------------------------------
@@ -220,6 +232,7 @@ The API returns the following response:
220232
"[max_age: 7d]": false,
221233
"[max_docs: 1000]": true,
222234
"[max_size: 5gb]": false,
235+
"[max_single_primary_size: 2gb]": false
223236
}
224237
}
225238
--------------------------------------------------
@@ -252,7 +265,8 @@ POST /my-data-stream/_rollover <2>
252265
"conditions" : {
253266
"max_age": "7d",
254267
"max_docs": 1000,
255-
"max_size": "5gb"
268+
"max_size": "5gb",
269+
"max_single_primary_size": "2gb"
256270
}
257271
}
258272
--------------------------------------------------
@@ -286,6 +300,7 @@ The API returns the following response:
286300
"[max_age: 7d]": false,
287301
"[max_docs: 1000]": true,
288302
"[max_size: 5gb]": false,
303+
"[max_single_primary_size: 2gb]": false
289304
}
290305
}
291306
--------------------------------------------------
@@ -332,7 +347,8 @@ POST /logs_write/_rollover
332347
"conditions" : {
333348
"max_age": "7d",
334349
"max_docs": 1000,
335-
"max_size": "5gb"
350+
"max_size": "5gb",
351+
"max_single_primary_size": "2gb"
336352
},
337353
"settings": {
338354
"index.number_of_shards": 2
@@ -359,7 +375,8 @@ POST /my_alias/_rollover/my_new_index_name
359375
"conditions": {
360376
"max_age": "7d",
361377
"max_docs": 1000,
362-
"max_size": "5gb"
378+
"max_size": "5gb",
379+
"max_single_primary_size": "2gb"
363380
}
364381
}
365382
--------------------------------------------------
@@ -457,7 +474,8 @@ POST /logs_write/_rollover?dry_run
457474
"conditions" : {
458475
"max_age": "7d",
459476
"max_docs": 1000,
460-
"max_size": "5gb"
477+
"max_size": "5gb",
478+
"max_single_primary_size": "2gb"
461479
}
462480
}
463481
--------------------------------------------------

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
---
2+
"Rollover with max_single_primary_size condition":
3+
- skip:
4+
version: " - 7.11.99"
5+
reason: max_single_primary_size condition was introduced in 7.12.0
6+
7+
# create index with alias and replica
8+
- do:
9+
indices.create:
10+
index: logs-1
11+
wait_for_active_shards: 1
12+
body:
13+
aliases:
14+
logs_search: {}
15+
16+
# index a document
17+
- do:
18+
index:
19+
index: logs-1
20+
id: "1"
21+
body: { "foo": "hello world" }
22+
refresh: true
23+
24+
# perform alias rollover with a large max_single_primary_size, no action.
25+
- do:
26+
indices.rollover:
27+
alias: "logs_search"
28+
wait_for_active_shards: 1
29+
body:
30+
conditions:
31+
max_single_primary_size: 100mb
32+
33+
- match: { conditions: { "[max_single_primary_size: 100mb]": false } }
34+
- match: { rolled_over: false }
35+
36+
# perform alias rollover with a small max_single_primary_size, got action.
37+
- do:
38+
indices.rollover:
39+
alias: "logs_search"
40+
wait_for_active_shards: 1
41+
body:
42+
conditions:
43+
max_single_primary_size: 10b
44+
45+
- match: { conditions: { "[max_single_primary_size: 10b]": true } }
46+
- match: { rolled_over: true }
47+
48+
# perform alias rollover on an empty index, no action.
49+
- do:
50+
indices.rollover:
51+
alias: "logs_search"
52+
wait_for_active_shards: 1
53+
body:
54+
conditions:
55+
max_single_primary_size: 1b
56+
57+
- match: { conditions: { "[max_single_primary_size: 1b]": false } }
58+
- match: { rolled_over: false }

‎server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,62 @@ public void testRolloverMaxSize() throws Exception {
435435
}
436436
}
437437

438+
public void testRolloverMaxSinglePrimarySize() throws Exception {
439+
assertAcked(prepareCreate("test-1").addAlias(new Alias("test_alias")).get());
440+
int numDocs = randomIntBetween(10, 20);
441+
for (int i = 0; i < numDocs; i++) {
442+
index("test-1", "doc", Integer.toString(i), "field", "foo-" + i);
443+
}
444+
flush("test-1");
445+
refresh("test_alias");
446+
447+
// A large max_single_primary_size
448+
{
449+
final RolloverResponse response = client().admin().indices()
450+
.prepareRolloverIndex("test_alias")
451+
.addMaxSinglePrimarySizeCondition(new ByteSizeValue(randomIntBetween(100, 50 * 1024), ByteSizeUnit.MB))
452+
.get();
453+
assertThat(response.getOldIndex(), equalTo("test-1"));
454+
assertThat(response.getNewIndex(), equalTo("test-000002"));
455+
assertThat("No rollover with a large max_single_primary_size condition", response.isRolledOver(), equalTo(false));
456+
final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-1");
457+
assertThat(oldIndex.getRolloverInfos().size(), equalTo(0));
458+
}
459+
460+
// A small max_single_primary_size
461+
{
462+
ByteSizeValue maxSinglePrimarySizeCondition = new ByteSizeValue(randomIntBetween(1, 20), ByteSizeUnit.BYTES);
463+
long beforeTime = client().threadPool().absoluteTimeInMillis() - 1000L;
464+
final RolloverResponse response = client().admin().indices()
465+
.prepareRolloverIndex("test_alias")
466+
.addMaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition)
467+
.get();
468+
assertThat(response.getOldIndex(), equalTo("test-1"));
469+
assertThat(response.getNewIndex(), equalTo("test-000002"));
470+
assertThat("Should rollover with a small max_single_primary_size condition", response.isRolledOver(), equalTo(true));
471+
final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-1");
472+
List<Condition<?>> metConditions = oldIndex.getRolloverInfos().get("test_alias").getMetConditions();
473+
assertThat(metConditions.size(), equalTo(1));
474+
assertThat(metConditions.get(0).toString(),
475+
equalTo(new MaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition).toString()));
476+
assertThat(oldIndex.getRolloverInfos().get("test_alias").getTime(),
477+
is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L))));
478+
}
479+
480+
// An empty index
481+
{
482+
final RolloverResponse response = client().admin().indices()
483+
.prepareRolloverIndex("test_alias")
484+
.addMaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES))
485+
.get();
486+
assertThat(response.getOldIndex(), equalTo("test-000002"));
487+
assertThat(response.getNewIndex(), equalTo("test-000003"));
488+
assertThat("No rollover with an empty index", response.isRolledOver(), equalTo(false));
489+
final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-000002");
490+
assertThat(oldIndex.getRolloverInfos().size(), equalTo(0));
491+
}
492+
}
493+
438494
public void testRejectIfAliasFoundInTemplate() throws Exception {
439495
client().admin().indices().preparePutTemplate("logs")
440496
.setPatterns(Collections.singletonList("logs-*")).addAlias(new Alias("logs-write")).get();

‎server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,13 @@ public static class Stats {
7575
public final long numDocs;
7676
public final long indexCreated;
7777
public final ByteSizeValue indexSize;
78+
public final ByteSizeValue maxSinglePrimarySize;
7879

79-
public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) {
80+
public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize, ByteSizeValue maxSinglePrimarySize) {
8081
this.numDocs = numDocs;
8182
this.indexCreated = indexCreated;
8283
this.indexSize = indexSize;
84+
this.maxSinglePrimarySize = maxSinglePrimarySize;
8385
}
8486
}
8587

0 commit comments

Comments
 (0)