Skip to content

Commit 5f89f8b

Browse files
Introduce Next Field in Paginated GetSnapshots Response (#74236)
Follow up to #73952 adding documentation for the `after` query parameter and the related `next` response field.
1 parent 49baa06 commit 5f89f8b

File tree

11 files changed

+351
-109
lines changed

11 files changed

+351
-109
lines changed

docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc

Lines changed: 142 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ PUT /_snapshot/my_repository
1919
2020
PUT /_snapshot/my_repository/my_snapshot?wait_for_completion=true
2121
22+
PUT /_snapshot/my_repository/snapshot_1?wait_for_completion=true
2223
PUT /_snapshot/my_repository/snapshot_2?wait_for_completion=true
24+
PUT /_snapshot/my_repository/snapshot_3?wait_for_completion=true
2325
----
2426
// TESTSETUP
2527
////
@@ -128,7 +130,15 @@ Allows setting a sort order for the result. Defaults to `start_time`, i.e. sorti
128130
(Optional, string)
129131
Sort order. Valid values are `asc` for ascending and `desc` for descending order. Defaults to `asc`, meaning ascending order.
130132

131-
NOTE: The pagination parameters `size`, `order`, and `sort` are not supported when using `verbose=false` and the sort order for
133+
`after`::
134+
(Optional, string)
135+
Offset identifier to start pagination from as returned by the `next` field in the response body.
136+
137+
NOTE: The `after` parameter and `next` field allow for iterating through snapshots with some consistency guarantees regarding concurrent
138+
creation or deletion of snapshots. It is guaranteed that any snapshot that exists at the beginning of the iteration and not concurrently
139+
deleted will be seen during the iteration. Snapshots concurrently created may be seen during an iteration.
140+
141+
NOTE: The pagination parameters `size`, `order`, `after` and `sort` are not supported when using `verbose=false` and the sort order for
132142
requests with `verbose=false` is undefined.
133143

134144
[role="child_attributes"]
@@ -268,6 +278,10 @@ The snapshot `state` can be one of the following values:
268278
that were not processed correctly.
269279
====
270280
--
281+
`next`::
282+
(string)
283+
If the request contained a size limit and there might be more results, a `next` field will be added to the response and can be used as the
284+
`after` query parameter to fetch additional results.
271285

272286
[[get-snapshot-api-example]]
273287
==== {api-examples-title}
@@ -319,3 +333,130 @@ The API returns the following response:
319333
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
320334
// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
321335
// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]
336+
337+
The following request returns information for all snapshots with prefix `snapshot` in the `my_repository` repository,
338+
limiting the response size to 2 and sorting by snapshot name.
339+
340+
[source,console]
341+
----
342+
GET /_snapshot/my_repository/snapshot*?size=2&sort=name
343+
----
344+
345+
The API returns the following response:
346+
347+
[source,console-result]
348+
----
349+
{
350+
"snapshots": [
351+
{
352+
"snapshot": "snapshot_1",
353+
"uuid": "dKb54xw67gvdRctLCxSket",
354+
"repository": "my_repository",
355+
"version_id": <version_id>,
356+
"version": <version>,
357+
"indices": [],
358+
"data_streams": [],
359+
"feature_states": [],
360+
"include_global_state": true,
361+
"state": "SUCCESS",
362+
"start_time": "2020-07-06T21:55:18.129Z",
363+
"start_time_in_millis": 1593093628850,
364+
"end_time": "2020-07-06T21:55:18.129Z",
365+
"end_time_in_millis": 1593094752018,
366+
"duration_in_millis": 0,
367+
"failures": [],
368+
"shards": {
369+
"total": 0,
370+
"failed": 0,
371+
"successful": 0
372+
}
373+
},
374+
{
375+
"snapshot": "snapshot_2",
376+
"uuid": "vdRctLCxSketdKb54xw67g",
377+
"repository": "my_repository",
378+
"version_id": <version_id>,
379+
"version": <version>,
380+
"indices": [],
381+
"data_streams": [],
382+
"feature_states": [],
383+
"include_global_state": true,
384+
"state": "SUCCESS",
385+
"start_time": "2020-07-06T21:55:18.130Z",
386+
"start_time_in_millis": 1593093628851,
387+
"end_time": "2020-07-06T21:55:18.130Z",
388+
"end_time_in_millis": 1593094752019,
389+
"duration_in_millis": 1,
390+
"failures": [],
391+
"shards": {
392+
"total": 0,
393+
"failed": 0,
394+
"successful": 0
395+
}
396+
}
397+
],
398+
"next": "c25hcHNob3RfMixteV9yZXBvc2l0b3J5LHNuYXBzaG90XzI="
399+
}
400+
----
401+
// TESTRESPONSE[s/"uuid": "dKb54xw67gvdRctLCxSket"/"uuid": $body.snapshots.0.uuid/]
402+
// TESTRESPONSE[s/"uuid": "vdRctLCxSketdKb54xw67g"/"uuid": $body.snapshots.1.uuid/]
403+
// TESTRESPONSE[s/"version_id": <version_id>/"version_id": $body.snapshots.0.version_id/]
404+
// TESTRESPONSE[s/"version": <version>/"version": $body.snapshots.0.version/]
405+
// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/]
406+
// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.130Z"/"start_time": $body.snapshots.1.start_time/]
407+
// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/]
408+
// TESTRESPONSE[s/"start_time_in_millis": 1593093628851/"start_time_in_millis": $body.snapshots.1.start_time_in_millis/]
409+
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
410+
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.130Z"/"end_time": $body.snapshots.1.end_time/]
411+
// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
412+
// TESTRESPONSE[s/"end_time_in_millis": 1593094752019/"end_time_in_millis": $body.snapshots.1.end_time_in_millis/]
413+
// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]
414+
// TESTRESPONSE[s/"duration_in_millis": 1/"duration_in_millis": $body.snapshots.1.duration_in_millis/]
415+
416+
A subsequent request for the remaining snapshots can then be made using the `next` value from the previous response as `after` parameter.
417+
418+
[source,console]
419+
----
420+
GET /_snapshot/my_repository/snapshot*?size=2&sort=name&after=c25hcHNob3RfMixteV9yZXBvc2l0b3J5LHNuYXBzaG90XzI=
421+
----
422+
423+
The API returns the following response:
424+
425+
[source,console-result]
426+
----
427+
{
428+
"snapshots": [
429+
{
430+
"snapshot": "snapshot_3",
431+
"uuid": "dRctdKb54xw67gvLCxSket",
432+
"repository": "my_repository",
433+
"version_id": <version_id>,
434+
"version": <version>,
435+
"indices": [],
436+
"data_streams": [],
437+
"feature_states": [],
438+
"include_global_state": true,
439+
"state": "SUCCESS",
440+
"start_time": "2020-07-06T21:55:18.129Z",
441+
"start_time_in_millis": 1593093628850,
442+
"end_time": "2020-07-06T21:55:18.129Z",
443+
"end_time_in_millis": 1593094752018,
444+
"duration_in_millis": 0,
445+
"failures": [],
446+
"shards": {
447+
"total": 0,
448+
"failed": 0,
449+
"successful": 0
450+
}
451+
}
452+
]
453+
}
454+
----
455+
// TESTRESPONSE[s/"uuid": "dRctdKb54xw67gvLCxSket"/"uuid": $body.snapshots.0.uuid/]
456+
// TESTRESPONSE[s/"version_id": <version_id>/"version_id": $body.snapshots.0.version_id/]
457+
// TESTRESPONSE[s/"version": <version>/"version": $body.snapshots.0.version/]
458+
// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/]
459+
// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/]
460+
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
461+
// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
462+
// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]

qa/smoke-test-http/src/test/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java

Lines changed: 35 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
2121
import org.elasticsearch.common.xcontent.XContentParser;
2222
import org.elasticsearch.common.xcontent.json.JsonXContent;
23+
import org.elasticsearch.core.Tuple;
2324
import org.elasticsearch.search.sort.SortOrder;
2425
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
2526
import org.elasticsearch.snapshots.SnapshotInfo;
@@ -64,8 +65,7 @@ public void testSortOrder() throws Exception {
6465
}
6566

6667
private void doTestSortOrder(String repoName, Collection<String> allSnapshotNames, SortOrder order) throws IOException {
67-
final List<SnapshotInfo> defaultSorting =
68-
clusterAdmin().prepareGetSnapshots(repoName).setOrder(order).get().getSnapshots();
68+
final List<SnapshotInfo> defaultSorting = clusterAdmin().prepareGetSnapshots(repoName).setOrder(order).get().getSnapshots();
6969
assertSnapshotListSorted(defaultSorting, null, order);
7070
assertSnapshotListSorted(
7171
allSnapshotsSorted(allSnapshotNames, repoName, GetSnapshotsRequest.SortBy.NAME, order),
@@ -106,24 +106,31 @@ private void doTestPagination(String repoName,
106106
GetSnapshotsRequest.SortBy sort,
107107
SortOrder order) throws IOException {
108108
final List<SnapshotInfo> allSnapshotsSorted = allSnapshotsSorted(names, repoName, sort, order);
109-
final List<SnapshotInfo> batch1 = sortedWithLimit(repoName, sort, 2, order);
110-
assertEquals(batch1, allSnapshotsSorted.subList(0, 2));
111-
final List<SnapshotInfo> batch2 = sortedWithLimit(repoName, sort, batch1.get(1), 2, order);
112-
assertEquals(batch2, allSnapshotsSorted.subList(2, 4));
113-
final int lastBatch = names.size() - batch1.size() - batch2.size();
114-
final List<SnapshotInfo> batch3 = sortedWithLimit(repoName, sort, batch2.get(1), lastBatch, order);
115-
assertEquals(batch3, allSnapshotsSorted.subList(batch1.size() + batch2.size(), names.size()));
116-
final List<SnapshotInfo> batch3NoLimit =
117-
sortedWithLimit(repoName, sort, batch2.get(1), GetSnapshotsRequest.NO_LIMIT, order);
118-
assertEquals(batch3, batch3NoLimit);
119-
final List<SnapshotInfo> batch3LargeLimit = sortedWithLimit(
109+
final Tuple<String, List<SnapshotInfo>> batch1 = sortedWithLimit(repoName, sort, null, 2, order);
110+
assertEquals(allSnapshotsSorted.subList(0, 2), batch1.v2());
111+
final Tuple<String, List<SnapshotInfo>> batch2 = sortedWithLimit(repoName, sort, batch1.v1(), 2, order);
112+
assertEquals(allSnapshotsSorted.subList(2, 4), batch2.v2());
113+
final int lastBatch = names.size() - batch1.v2().size() - batch2.v2().size();
114+
final Tuple<String, List<SnapshotInfo>> batch3 = sortedWithLimit(repoName, sort, batch2.v1(), lastBatch, order);
115+
assertEquals(batch3.v2(), allSnapshotsSorted.subList(batch1.v2().size() + batch2.v2().size(), names.size()));
116+
final Tuple<String, List<SnapshotInfo>> batch3NoLimit = sortedWithLimit(
120117
repoName,
121118
sort,
122-
batch2.get(1),
119+
batch2.v1(),
120+
GetSnapshotsRequest.NO_LIMIT,
121+
order
122+
);
123+
assertNull(batch3NoLimit.v1());
124+
assertEquals(batch3.v2(), batch3NoLimit.v2());
125+
final Tuple<String, List<SnapshotInfo>> batch3LargeLimit = sortedWithLimit(
126+
repoName,
127+
sort,
128+
batch2.v1(),
123129
lastBatch + randomIntBetween(1, 100),
124130
order
125131
);
126-
assertEquals(batch3, batch3LargeLimit);
132+
assertEquals(batch3.v2(), batch3LargeLimit.v2());
133+
assertNull(batch3LargeLimit.v1());
127134
}
128135

129136
public void testSortAndPaginateWithInProgress() throws Exception {
@@ -173,14 +180,15 @@ private static void assertStablePagination(String repoName,
173180
final List<SnapshotInfo> allSorted = allSnapshotsSorted(allSnapshotNames, repoName, sort, order);
174181

175182
for (int i = 1; i <= allSnapshotNames.size(); i++) {
176-
final List<SnapshotInfo> subsetSorted = sortedWithLimit(repoName, sort, i, order);
183+
final List<SnapshotInfo> subsetSorted = sortedWithLimit(repoName, sort, null, i, order).v2();
177184
assertEquals(subsetSorted, allSorted.subList(0, i));
178185
}
179186

180187
for (int j = 0; j < allSnapshotNames.size(); j++) {
181188
final SnapshotInfo after = allSorted.get(j);
182189
for (int i = 1; i < allSnapshotNames.size() - j; i++) {
183-
final List<SnapshotInfo> subsetSorted = sortedWithLimit(repoName, sort, after, i, order);
190+
final List<SnapshotInfo> subsetSorted = sortedWithLimit(
191+
repoName, sort, GetSnapshotsRequest.After.from(after, sort).asQueryParam(), i, order).v2();
184192
assertEquals(subsetSorted, allSorted.subList(j + 1, j + i + 1));
185193
}
186194
}
@@ -196,7 +204,7 @@ private static List<SnapshotInfo> allSnapshotsSorted(Collection<String> allSnaps
196204
request.addParameter("order", order.toString());
197205
}
198206
final Response response = getRestClient().performRequest(request);
199-
final List<SnapshotInfo> snapshotInfos = readSnapshotInfos(response);
207+
final List<SnapshotInfo> snapshotInfos = readSnapshotInfos(response).v2();
200208
assertEquals(snapshotInfos.size(), allSnapshotNames.size());
201209
for (SnapshotInfo snapshotInfo : snapshotInfos) {
202210
assertThat(snapshotInfo.snapshotId().getName(), is(in(allSnapshotNames)));
@@ -208,42 +216,27 @@ private static Request baseGetSnapshotsRequest(String repoName) {
208216
return new Request(HttpGet.METHOD_NAME, "/_snapshot/" + repoName + "/*");
209217
}
210218

211-
private static List<SnapshotInfo> sortedWithLimit(String repoName,
212-
GetSnapshotsRequest.SortBy sortBy,
213-
int size,
214-
SortOrder order) throws IOException {
215-
final Request request = baseGetSnapshotsRequest(repoName);
216-
request.addParameter("sort", sortBy.toString());
217-
if (order == SortOrder.DESC || randomBoolean()) {
218-
request.addParameter("order", order.toString());
219-
}
220-
request.addParameter("size", String.valueOf(size));
221-
final Response response = getRestClient().performRequest(request);
222-
return readSnapshotInfos(response);
223-
}
224-
225-
private static List<SnapshotInfo> readSnapshotInfos(Response response) throws IOException {
226-
final List<SnapshotInfo> snapshotInfos;
219+
private static Tuple<String, List<SnapshotInfo>> readSnapshotInfos(Response response) throws IOException {
227220
try (InputStream input = response.getEntity().getContent();
228221
XContentParser parser = JsonXContent.jsonXContent.createParser(
229222
NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, input)) {
230-
snapshotInfos = GetSnapshotsResponse.fromXContent(parser).getSnapshots();
223+
final GetSnapshotsResponse getSnapshotsResponse = GetSnapshotsResponse.fromXContent(parser);
224+
return Tuple.tuple(getSnapshotsResponse.next(), getSnapshotsResponse.getSnapshots());
231225
}
232-
return snapshotInfos;
233226
}
234227

235-
private static List<SnapshotInfo> sortedWithLimit(String repoName,
236-
GetSnapshotsRequest.SortBy sortBy,
237-
SnapshotInfo after,
238-
int size,
239-
SortOrder order) throws IOException {
228+
private static Tuple<String, List<SnapshotInfo>> sortedWithLimit(String repoName,
229+
GetSnapshotsRequest.SortBy sortBy,
230+
String after,
231+
int size,
232+
SortOrder order) throws IOException {
240233
final Request request = baseGetSnapshotsRequest(repoName);
241234
request.addParameter("sort", sortBy.toString());
242235
if (size != GetSnapshotsRequest.NO_LIMIT || randomBoolean()) {
243236
request.addParameter("size", String.valueOf(size));
244237
}
245238
if (after != null) {
246-
request.addParameter("after", GetSnapshotsRequest.After.from(after, sortBy).value() + "," + after.snapshotId().getName());
239+
request.addParameter("after", after);
247240
}
248241
if (order == SortOrder.DESC || randomBoolean()) {
249242
request.addParameter("order", order.toString());

0 commit comments

Comments
 (0)