Skip to content

Commit ffaa0f2

Browse files
original-brownbearoriginal-brownbear
authored
Implement Numeric Offset Parameter in Get Snapshots API (#76233)
Add numeric offset parameter to this API. Relates #74350
1 parent 862b375 commit ffaa0f2

File tree

8 files changed

+158
-25
lines changed

8 files changed

+158
-25
lines changed

‎docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,12 +134,17 @@ Sort order. Valid values are `asc` for ascending and `desc` for descending order
134134
(Optional, string)
135135
Offset identifier to start pagination from as returned by the `next` field in the response body.
136136

137+
`offset`::
138+
(Optional, integer)
139+
Numeric offset to start pagination from based on the snapshots matching this request. Using a non-zero value for this parameter is mutually
140+
exclusive with using the `after` parameter. Defaults to `0`.
141+
137142
NOTE: The `after` parameter and `next` field allow for iterating through snapshots with some consistency guarantees regarding concurrent
138143
creation or deletion of snapshots. It is guaranteed that any snapshot that exists at the beginning of the iteration and not concurrently
139144
deleted will be seen during the iteration. Snapshots concurrently created may be seen during an iteration.
140145

141-
NOTE: The pagination parameters `size`, `order`, `after` and `sort` are not supported when using `verbose=false` and the sort order for
142-
requests with `verbose=false` is undefined.
146+
NOTE: The pagination parameters `size`, `order`, `after`, `offset` and `sort` are not supported when using `verbose=false` and the sort
147+
order for requests with `verbose=false` is undefined.
143148

144149
[role="child_attributes"]
145150
[[get-snapshot-api-response-body]]
@@ -435,6 +440,56 @@ GET /_snapshot/my_repository/snapshot*?size=2&sort=name&after=c25hcHNob3RfMixteV
435440

436441
The API returns the following response:
437442

443+
[source,console-result]
444+
----
445+
{
446+
"snapshots": [
447+
{
448+
"snapshot": "snapshot_3",
449+
"uuid": "dRctdKb54xw67gvLCxSket",
450+
"repository": "my_repository",
451+
"version_id": <version_id>,
452+
"version": <version>,
453+
"indices": [],
454+
"data_streams": [],
455+
"feature_states": [],
456+
"include_global_state": true,
457+
"state": "SUCCESS",
458+
"start_time": "2020-07-06T21:55:18.129Z",
459+
"start_time_in_millis": 1593093628850,
460+
"end_time": "2020-07-06T21:55:18.129Z",
461+
"end_time_in_millis": 1593094752018,
462+
"duration_in_millis": 0,
463+
"failures": [],
464+
"shards": {
465+
"total": 0,
466+
"failed": 0,
467+
"successful": 0
468+
}
469+
}
470+
],
471+
"total": 3,
472+
"remaining": 0
473+
}
474+
----
475+
// TESTRESPONSE[s/"uuid": "dRctdKb54xw67gvLCxSket"/"uuid": $body.snapshots.0.uuid/]
476+
// TESTRESPONSE[s/"version_id": <version_id>/"version_id": $body.snapshots.0.version_id/]
477+
// TESTRESPONSE[s/"version": <version>/"version": $body.snapshots.0.version/]
478+
// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/]
479+
// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/]
480+
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
481+
// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
482+
// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]
483+
484+
Alternatively, the same result could be retrieved by using an offset value of `2` to skip the two snapshot already seen.
485+
486+
[source,console]
487+
----
488+
GET /_snapshot/my_repository/snapshot*?size=2&sort=name&offset=2
489+
----
490+
491+
The API returns the following response:
492+
438493
[source,console-result]
439494
----
440495
{

‎qa/smoke-test-http/src/test/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -191,10 +191,14 @@ private static void assertStablePagination(String repoName,
191191
for (int i = 1; i < allSnapshotNames.size() - j; i++) {
192192
final GetSnapshotsResponse getSnapshotsResponse =
193193
sortedWithLimit(repoName, sort, GetSnapshotsRequest.After.from(after, sort).asQueryParam(), i, order);
194+
final GetSnapshotsResponse getSnapshotsResponseNumeric = sortedWithLimit(repoName, sort, j + 1, i, order);
194195
final List<SnapshotInfo> subsetSorted = getSnapshotsResponse.getSnapshots();
196+
assertEquals(subsetSorted, getSnapshotsResponseNumeric.getSnapshots());
195197
assertEquals(subsetSorted, allSorted.subList(j + 1, j + i + 1));
196198
assertEquals(allSnapshotNames.size(), getSnapshotsResponse.totalCount());
197199
assertEquals(allSnapshotNames.size() - (j + i + 1), getSnapshotsResponse.remaining());
200+
assertEquals(getSnapshotsResponseNumeric.totalCount(), getSnapshotsResponse.totalCount());
201+
assertEquals(getSnapshotsResponseNumeric.remaining(), getSnapshotsResponse.remaining());
198202
}
199203
}
200204
}
@@ -232,10 +236,10 @@ private static GetSnapshotsResponse readSnapshotInfos(Response response) throws
232236
}
233237

234238
private static GetSnapshotsResponse sortedWithLimit(String repoName,
235-
GetSnapshotsRequest.SortBy sortBy,
236-
String after,
237-
int size,
238-
SortOrder order) throws IOException {
239+
GetSnapshotsRequest.SortBy sortBy,
240+
String after,
241+
int size,
242+
SortOrder order) throws IOException {
239243
final Request request = baseGetSnapshotsRequest(repoName);
240244
request.addParameter("sort", sortBy.toString());
241245
if (size != GetSnapshotsRequest.NO_LIMIT || randomBoolean()) {
@@ -250,4 +254,22 @@ private static GetSnapshotsResponse sortedWithLimit(String repoName,
250254
final Response response = getRestClient().performRequest(request);
251255
return readSnapshotInfos(response);
252256
}
257+
258+
private static GetSnapshotsResponse sortedWithLimit(String repoName,
259+
GetSnapshotsRequest.SortBy sortBy,
260+
int offset,
261+
int size,
262+
SortOrder order) throws IOException {
263+
final Request request = baseGetSnapshotsRequest(repoName);
264+
request.addParameter("sort", sortBy.toString());
265+
if (size != GetSnapshotsRequest.NO_LIMIT || randomBoolean()) {
266+
request.addParameter("size", String.valueOf(size));
267+
}
268+
request.addParameter("offset", String.valueOf(offset));
269+
if (order == SortOrder.DESC || randomBoolean()) {
270+
request.addParameter("order", order.toString());
271+
}
272+
final Response response = getRestClient().performRequest(request);
273+
return readSnapshotInfos(response);
274+
}
253275
}

‎server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -197,11 +197,15 @@ private static void assertStablePagination(String repoName, Collection<String> a
197197
i,
198198
order
199199
);
200+
final GetSnapshotsResponse getSnapshotsResponseNumeric = sortedWithLimit(repoName, sort, j + 1, i, order);
200201
final List<SnapshotInfo> subsetSorted = getSnapshotsResponse.getSnapshots();
202+
assertEquals(subsetSorted, getSnapshotsResponseNumeric.getSnapshots());
201203
assertEquals(subsetSorted, allSorted.subList(j + 1, j + i + 1));
202204
assertEquals(allSnapshotNames.size(), getSnapshotsResponse.totalCount());
203205
assertEquals(allSnapshotNames.size() - (j + i + 1), getSnapshotsResponse.remaining());
204206
assertEquals(subsetSorted, allSorted.subList(j + 1, j + i + 1));
207+
assertEquals(getSnapshotsResponseNumeric.totalCount(), getSnapshotsResponse.totalCount());
208+
assertEquals(getSnapshotsResponseNumeric.remaining(), getSnapshotsResponse.remaining());
205209
}
206210
}
207211
}
@@ -230,12 +234,17 @@ private static GetSnapshotsResponse sortedWithLimit(
230234
int size,
231235
SortOrder order
232236
) {
233-
final GetSnapshotsResponse response = baseGetSnapshotsRequest(repoName).setAfter(after)
234-
.setSort(sortBy)
235-
.setSize(size)
236-
.setOrder(order)
237-
.get();
238-
return response;
237+
return baseGetSnapshotsRequest(repoName).setAfter(after).setSort(sortBy).setSize(size).setOrder(order).get();
238+
}
239+
240+
private static GetSnapshotsResponse sortedWithLimit(
241+
String repoName,
242+
GetSnapshotsRequest.SortBy sortBy,
243+
int offset,
244+
int size,
245+
SortOrder order
246+
) {
247+
return baseGetSnapshotsRequest(repoName).setOffset(offset).setSort(sortBy).setSize(size).setOrder(order).get();
239248
}
240249

241250
private static GetSnapshotsRequestBuilder baseGetSnapshotsRequest(String repoName) {

‎server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
5252
*/
5353
private int size = NO_LIMIT;
5454

55+
/**
56+
* Numeric offset at which to start fetching snapshots. Mutually exclusive with {@link After} if not equal to {@code 0}.
57+
*/
58+
private int offset = 0;
59+
5560
@Nullable
5661
private After after;
5762

@@ -104,6 +109,9 @@ public GetSnapshotsRequest(StreamInput in) throws IOException {
104109
sort = in.readEnum(SortBy.class);
105110
size = in.readVInt();
106111
order = SortOrder.readFromStream(in);
112+
if (in.getVersion().onOrAfter(NUMERIC_PAGINATION_VERSION)) {
113+
offset = in.readVInt();
114+
}
107115
}
108116
}
109117

@@ -130,6 +138,13 @@ public void writeTo(StreamOutput out) throws IOException {
130138
out.writeEnum(sort);
131139
out.writeVInt(size);
132140
order.writeTo(out);
141+
if (out.getVersion().onOrAfter(NUMERIC_PAGINATION_VERSION)) {
142+
out.writeVInt(offset);
143+
} else if (offset != 0) {
144+
throw new IllegalArgumentException(
145+
"can't use numeric offset in get snapshots request with node version [" + out.getVersion() + "]"
146+
);
147+
}
133148
} else if (sort != SortBy.START_TIME || size != NO_LIMIT || after != null || order != SortOrder.ASC) {
134149
throw new IllegalArgumentException("can't use paginated get snapshots request with node version [" + out.getVersion() + "]");
135150
}
@@ -151,12 +166,17 @@ public ActionRequestValidationException validate() {
151166
if (size > 0) {
152167
validationException = addValidationError("can't use size limit with verbose=false", validationException);
153168
}
169+
if (offset > 0) {
170+
validationException = addValidationError("can't use offset with verbose=false", validationException);
171+
}
154172
if (after != null) {
155173
validationException = addValidationError("can't use after with verbose=false", validationException);
156174
}
157175
if (order != SortOrder.ASC) {
158176
validationException = addValidationError("can't use non-default sort order with verbose=false", validationException);
159177
}
178+
} else if (after != null && offset > 0) {
179+
validationException = addValidationError("can't use after and offset simultaneously", validationException);
160180
}
161181
return validationException;
162182
}
@@ -265,6 +285,15 @@ public int size() {
265285
return size;
266286
}
267287

288+
public int offset() {
289+
return offset;
290+
}
291+
292+
public GetSnapshotsRequest offset(int offset) {
293+
this.offset = offset;
294+
return this;
295+
}
296+
268297
public SortOrder order() {
269298
return order;
270299
}

‎server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ public GetSnapshotsRequestBuilder setSize(int size) {
116116
return this;
117117
}
118118

119+
public GetSnapshotsRequestBuilder setOffset(int offset) {
120+
request.offset(offset);
121+
return this;
122+
}
123+
119124
public GetSnapshotsRequestBuilder setOrder(SortOrder order) {
120125
request.order(order);
121126
return this;

‎server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ protected void masterOperation(
117117
(CancellableTask) task,
118118
request.sort(),
119119
request.after(),
120+
request.offset(),
120121
request.size(),
121122
request.order(),
122123
listener
@@ -133,6 +134,7 @@ private void getMultipleReposSnapshotInfo(
133134
CancellableTask cancellableTask,
134135
GetSnapshotsRequest.SortBy sortBy,
135136
@Nullable GetSnapshotsRequest.After after,
137+
int offset,
136138
int size,
137139
SortOrder order,
138140
ActionListener<GetSnapshotsResponse> listener
@@ -154,7 +156,7 @@ private void getMultipleReposSnapshotInfo(
154156
.map(Tuple::v1)
155157
.filter(Objects::nonNull)
156158
.collect(Collectors.toMap(Tuple::v1, Tuple::v2));
157-
final SnapshotsInRepo snInfos = sortSnapshots(allSnapshots, sortBy, after, size, order);
159+
final SnapshotsInRepo snInfos = sortSnapshots(allSnapshots, sortBy, after, offset, size, order);
158160
final List<SnapshotInfo> snapshotInfos = snInfos.snapshotInfos;
159161
final int remaining = snInfos.remaining + responses.stream()
160162
.map(Tuple::v2)
@@ -183,7 +185,6 @@ private void getMultipleReposSnapshotInfo(
183185
cancellableTask,
184186
sortBy,
185187
after,
186-
size,
187188
order,
188189
groupedActionListener.delegateResponse((groupedListener, e) -> {
189190
if (isMultiRepoRequest && e instanceof ElasticsearchException) {
@@ -205,7 +206,6 @@ private void getSingleRepoSnapshotInfo(
205206
CancellableTask task,
206207
GetSnapshotsRequest.SortBy sortBy,
207208
@Nullable final GetSnapshotsRequest.After after,
208-
int size,
209209
SortOrder order,
210210
ActionListener<SnapshotsInRepo> listener
211211
) {
@@ -237,7 +237,6 @@ private void getSingleRepoSnapshotInfo(
237237
task,
238238
sortBy,
239239
after,
240-
size,
241240
order,
242241
listener
243242
),
@@ -277,7 +276,6 @@ private void loadSnapshotInfos(
277276
CancellableTask task,
278277
GetSnapshotsRequest.SortBy sortBy,
279278
@Nullable final GetSnapshotsRequest.After after,
280-
int size,
281279
SortOrder order,
282280
ActionListener<SnapshotsInRepo> listener
283281
) {
@@ -327,22 +325,22 @@ private void loadSnapshotInfos(
327325
task,
328326
sortBy,
329327
after,
330-
size,
331328
order,
332329
listener
333330
);
334331
} else {
335332
final SnapshotsInRepo snapshotInfos;
336333
if (repositoryData != null) {
337334
// want non-current snapshots as well, which are found in the repository data
338-
snapshotInfos = buildSimpleSnapshotInfos(toResolve, repo, repositoryData, currentSnapshots, sortBy, after, size, order);
335+
snapshotInfos = buildSimpleSnapshotInfos(toResolve, repo, repositoryData, currentSnapshots, sortBy, after, order);
339336
} else {
340337
// only want current snapshots
341338
snapshotInfos = sortSnapshots(
342339
currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList()),
343340
sortBy,
344341
after,
345-
size,
342+
0,
343+
GetSnapshotsRequest.NO_LIMIT,
346344
order
347345
);
348346
}
@@ -365,7 +363,6 @@ private void snapshots(
365363
CancellableTask task,
366364
GetSnapshotsRequest.SortBy sortBy,
367365
@Nullable GetSnapshotsRequest.After after,
368-
int size,
369366
SortOrder order,
370367
ActionListener<SnapshotsInRepo> listener
371368
) {
@@ -395,7 +392,7 @@ private void snapshots(
395392
final ActionListener<Void> allDoneListener = listener.delegateFailure((l, v) -> {
396393
final ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotInfos);
397394
snapshotList.addAll(snapshotSet);
398-
listener.onResponse(sortSnapshots(snapshotList, sortBy, after, size, order));
395+
listener.onResponse(sortSnapshots(snapshotList, sortBy, after, 0, GetSnapshotsRequest.NO_LIMIT, order));
399396
});
400397
if (snapshotIdsToIterate.isEmpty()) {
401398
allDoneListener.onResponse(null);
@@ -445,7 +442,6 @@ private static SnapshotsInRepo buildSimpleSnapshotInfos(
445442
final List<SnapshotInfo> currentSnapshots,
446443
final GetSnapshotsRequest.SortBy sortBy,
447444
@Nullable final GetSnapshotsRequest.After after,
448-
final int size,
449445
final SortOrder order
450446
) {
451447
List<SnapshotInfo> snapshotInfos = new ArrayList<>();
@@ -475,7 +471,7 @@ private static SnapshotsInRepo buildSimpleSnapshotInfos(
475471
)
476472
);
477473
}
478-
return sortSnapshots(snapshotInfos, sortBy, after, size, order);
474+
return sortSnapshots(snapshotInfos, sortBy, after, 0, GetSnapshotsRequest.NO_LIMIT, order);
479475
}
480476

481477
private static final Comparator<SnapshotInfo> BY_START_TIME = Comparator.comparingLong(SnapshotInfo::startTime)
@@ -494,6 +490,7 @@ private static SnapshotsInRepo sortSnapshots(
494490
final List<SnapshotInfo> snapshotInfos,
495491
final GetSnapshotsRequest.SortBy sortBy,
496492
final @Nullable GetSnapshotsRequest.After after,
493+
final int offset,
497494
final int size,
498495
final SortOrder order
499496
) {
@@ -518,6 +515,7 @@ private static SnapshotsInRepo sortSnapshots(
518515
Stream<SnapshotInfo> infos = snapshotInfos.stream();
519516

520517
if (after != null) {
518+
assert offset == 0 : "can't combine after and offset but saw [" + after + "] and offset [" + offset + "]";
521519
final Predicate<SnapshotInfo> isAfter;
522520
final String snapshotName = after.snapshotName();
523521
final String repoName = after.repoName();
@@ -553,7 +551,7 @@ private static SnapshotsInRepo sortSnapshots(
553551
}
554552
infos = infos.filter(isAfter);
555553
}
556-
infos = infos.sorted(order == SortOrder.DESC ? comparator.reversed() : comparator);
554+
infos = infos.sorted(order == SortOrder.DESC ? comparator.reversed() : comparator).skip(offset);
557555
final List<SnapshotInfo> allSnapshots = infos.collect(Collectors.toUnmodifiableList());
558556
final List<SnapshotInfo> snapshots;
559557
if (size != GetSnapshotsRequest.NO_LIMIT) {

0 commit comments

Comments
 (0)