Snapshot Pagination and Scalability Improvements Backport to 7.x (#74676)

Backport of the recently introduced snapshot pagination and scalability improvements listed below.
Merged as a single backport because the `7.x` and master snapshot status API logic had massively diverged between master and 7.x. With the work in the below PRs, the logic in master and 7.x once again has been aligned very closely again.

#72842
#73172
#73199
#73570 
#73952
#74236 
#74451 (this one is only partly applicable as it was mainly a change to master to align `master` and `7.x` branches)

Author: original-brownbear

client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java

@@ -33,22 +33,23 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.snapshots.RestoreInfo;
 import org.elasticsearch.snapshots.SnapshotInfo;
+import org.mockito.internal.util.collections.Sets;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE;
 import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME;
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 
 public class SnapshotIT extends ESRestHighLevelClientTestCase {
@@ -177,50 +178,54 @@ public void testCreateSnapshot() throws Exception {
     }
 
     public void testGetSnapshots() throws IOException {
-        String repository = "test_repository";
+        String repository1 = "test_repository1";
+        String repository2 = "test_repository2";
         String snapshot1 = "test_snapshot1";
         String snapshot2 = "test_snapshot2";
 
-        AcknowledgedResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}");
+        AcknowledgedResponse putRepositoryResponse =
+                createTestRepository(repository1, FsRepository.TYPE, "{\"location\": \"loc1\"}");
         assertTrue(putRepositoryResponse.isAcknowledged());
 
-        CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository, snapshot1);
+        AcknowledgedResponse putRepositoryResponse2 =
+                createTestRepository(repository2, FsRepository.TYPE, "{\"location\": \"loc2\"}");
+        assertTrue(putRepositoryResponse2.isAcknowledged());
+
+        CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository1, snapshot1);
         createSnapshotRequest1.waitForCompletion(true);
         CreateSnapshotResponse putSnapshotResponse1 = createTestSnapshot(createSnapshotRequest1);
-        CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository, snapshot2);
+        CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository2, snapshot2);
         createSnapshotRequest2.waitForCompletion(true);
-        Map<String, Object> originalMetadata = randomUserMetadata();
+        Map<String, Object> originalMetadata = AbstractSnapshotIntegTestCase.randomUserMetadata();
         createSnapshotRequest2.userMetadata(originalMetadata);
         CreateSnapshotResponse putSnapshotResponse2 = createTestSnapshot(createSnapshotRequest2);
         // check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead.
         assertEquals(RestStatus.OK, putSnapshotResponse1.status());
         assertEquals(RestStatus.OK, putSnapshotResponse2.status());
 
-        GetSnapshotsRequest request;
-        if (randomBoolean()) {
-            request = new GetSnapshotsRequest(repository);
-        } else if (randomBoolean()) {
-            request = new GetSnapshotsRequest(repository, new String[] {"_all"});
+        GetSnapshotsRequest request = new GetSnapshotsRequest(
+                randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{repository1, repository2}),
+                randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{snapshot1, snapshot2})
+        );
+        request.ignoreUnavailable(true);
 
-        } else {
-            request = new GetSnapshotsRequest(repository, new String[] {snapshot1, snapshot2});
-        }
         GetSnapshotsResponse response = execute(request, highLevelClient().snapshot()::get, highLevelClient().snapshot()::getAsync);
 
-        assertEquals(2, response.getSnapshots().size());
-        assertThat(response.getSnapshots().stream().map((s) -> s.snapshotId().getName()).collect(Collectors.toList()),
-            contains("test_snapshot1", "test_snapshot2"));
-        Optional<Map<String, Object>> returnedMetadata = response.getSnapshots().stream()
-            .filter(s -> s.snapshotId().getName().equals("test_snapshot2"))
-            .findFirst()
-            .map(SnapshotInfo::userMetadata);
-        if (returnedMetadata.isPresent()) {
-            assertEquals(originalMetadata, returnedMetadata.get());
-        } else {
-            assertNull("retrieved metadata is null, expected non-null metadata", originalMetadata);
-        }
+        assertThat(response.isFailed(), is(false));
+        assertEquals(
+            Sets.newSet(repository1, repository2),
+            response.getSnapshots().stream().map(SnapshotInfo::repository).collect(Collectors.toSet())
+        );
+
+        assertThat(response.getSnapshots(), hasSize(2));
+        assertThat(response.getSnapshots().get(0).snapshotId().getName(), equalTo(snapshot1));
+        assertThat(response.getSnapshots().get(0).repository(), equalTo(repository1));
+        assertThat(response.getSnapshots().get(1).snapshotId().getName(), equalTo(snapshot2));
+        assertThat(response.getSnapshots().get(1).userMetadata(), equalTo(originalMetadata));
+        assertThat(response.getSnapshots().get(1).repository(), equalTo(repository2));
     }
 
+
     public void testSnapshotsStatus() throws IOException {
         String testRepository = "test";
         String testSnapshot = "snapshot";

docs/reference/cat/snapshots.asciidoc

@@ -13,6 +13,8 @@ more repositories. A snapshot is a backup of an index or running {es} cluster.
 
 `GET /_cat/snapshots/<repository>`
 
+`GET /_cat/snapshots`
+
 [[cat-snapshots-api-prereqs]]
 ==== {api-prereq-title}
 
@@ -27,9 +29,10 @@ more repositories. A snapshot is a backup of an index or running {es} cluster.
 `<repository>`::
 +
 --
-(Required, string) Snapshot repository used to limit the request.
+(Optional, string) Comma-separated list of snapshot repositories used to limit
+the request. Accepts wildcard expressions. `_all` returns all repositories.
 
-If the repository fails during the request, {es} returns an error.
+If any repository fails during the request, {es} returns an error.
 --
 
 
@@ -127,9 +130,9 @@ The API returns the following response:
 
 [source,txt]
 --------------------------------------------------
-id     status start_epoch start_time end_epoch  end_time duration indices successful_shards failed_shards total_shards
-snap1  FAILED 1445616705  18:11:45   1445616978 18:16:18     4.6m       1                 4             1            5
-snap2 SUCCESS 1445634298  23:04:58   1445634672 23:11:12     6.2m       2                10             0           10
+id     repository status start_epoch start_time end_epoch  end_time duration indices successful_shards failed_shards total_shards
+snap1  repo1      FAILED 1445616705  18:11:45   1445616978 18:16:18     4.6m       1                 4             1            5
+snap2  repo1      SUCCESS 1445634298  23:04:58   1445634672 23:11:12     6.2m       2                10             0           10
 --------------------------------------------------
 // TESTRESPONSE[s/FAILED/SUCCESS/ s/14456\d+/\\d+/ s/\d+(\.\d+)?(m|s|ms)/\\d+(\\.\\d+)?(m|s|ms)/]
 // TESTRESPONSE[s/\d+:\d+:\d+/\\d+:\\d+:\\d+/]

docs/reference/modules/threadpool.asciidoc

@@ -43,6 +43,11 @@ There are several thread pools, but the important ones include:
     keep-alive of `5m` and a max of `min(5, (`<<node.processors,
     `# of allocated processors`>>`) / 2)`.
 
+`snapshot_meta`::
+    For snapshot repository metadata read operations. Thread pool type is `scaling` with a
+    keep-alive of `5m` and a max of `min(50, (`<<node.processors,
+    `# of allocated processors`>>` pass:[ * ]3))`.
+
 `warmer`::
     For segment warm-up operations. Thread pool type is `scaling` with a
     keep-alive of `5m` and a max of `min(5, (`<<node.processors,

docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc

@@ -176,6 +176,7 @@ The API returns the following response:
   "snapshot": {
     "snapshot": "snapshot_2",
     "uuid": "vdRctLCxSketdKb54xw67g",
+    "repository": "my_repository",
     "version_id": <version_id>,
     "version": <version>,
     "indices": [],

docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc

@@ -19,7 +19,9 @@ PUT /_snapshot/my_repository
 
 PUT /_snapshot/my_repository/my_snapshot?wait_for_completion=true
 
+PUT /_snapshot/my_repository/snapshot_1?wait_for_completion=true
 PUT /_snapshot/my_repository/snapshot_2?wait_for_completion=true
+PUT /_snapshot/my_repository/snapshot_3?wait_for_completion=true
 ----
 // TESTSETUP
 ////
@@ -57,7 +59,8 @@ Use the get snapshot API to return information about one or more snapshots, incl
 
 `<repository>`::
 (Required, string)
-Snapshot repository name used to limit the request.
+Comma-separated list of snapshot repository names used to limit the request.
+Wildcard (`*`) expressions are supported.
 +
 To get information about all snapshot repositories registered in the
 cluster, omit this parameter or use `*` or `_all`.
@@ -99,6 +102,45 @@ comprising the number of shards in the index, the total size of the index in
 bytes, and the maximum number of segments per shard in the index. Defaults to
 `false`, meaning that this information is omitted.
 
+`sort`::
+(Optional, string)
+Allows setting a sort order for the result. Defaults to `start_time`, i.e. sorting by snapshot start time stamp.
++
+.Valid values for `sort`
+[%collapsible%open]
+====
+`start_time`::
+  Sort snapshots by their start time stamp and break ties by snapshot name.
+
+`duration`::
+  Sort snapshots by their duration and break ties by snapshot name.
+
+`name`::
+  Sort snapshots by their name.
+
+`index_count`::
+  Sort snapshots by the number of indices they contain and break ties by snapshot name.
+====
+
+`size`::
+(Optional, integer)
+ Maximum number of snapshots to return. Defaults to `0` which means return all that match the request without limit.
+
+`order`::
+(Optional, string)
+Sort order. Valid values are `asc` for ascending and `desc` for descending order. Defaults to `asc`, meaning ascending order.
+
+`after`::
+(Optional, string)
+Offset identifier to start pagination from as returned by the `next` field in the response body.
+
+NOTE: The `after` parameter and `next` field allow for iterating through snapshots with some consistency guarantees regarding concurrent
+creation or deletion of snapshots. It is guaranteed that  any snapshot that exists at the beginning of the iteration and not concurrently
+deleted will be seen during the iteration. Snapshots concurrently created may be seen during an iteration.
+
+NOTE: The pagination parameters `size`, `order`, `after` and `sort` are not supported when using `verbose=false` and the sort order for
+requests with `verbose=false` is undefined.
+
 [role="child_attributes"]
 [[get-snapshot-api-response-body]]
 ==== {api-response-body-title}
@@ -236,6 +278,10 @@ The snapshot `state` can be one of the following values:
   that were not processed correctly.
 ====
 --
+`next`::
+(string)
+If the request contained a size limit and there might be more results, a `next` field will be added to the response and can be used as the
+`after` query parameter to fetch additional results.
 
 [[get-snapshot-api-example]]
 ==== {api-examples-title}
@@ -256,6 +302,7 @@ The API returns the following response:
     {
       "snapshot": "snapshot_2",
       "uuid": "vdRctLCxSketdKb54xw67g",
+      "repository": "my_repository",
       "version_id": <version_id>,
       "version": <version>,
       "indices": [],
@@ -265,7 +312,7 @@ The API returns the following response:
       "state": "SUCCESS",
       "start_time": "2020-07-06T21:55:18.129Z",
       "start_time_in_millis": 1593093628850,
-      "end_time": "2020-07-06T21:55:18.876Z",
+      "end_time": "2020-07-06T21:55:18.129Z",
       "end_time_in_millis": 1593094752018,
       "duration_in_millis": 0,
       "failures": [],
@@ -283,6 +330,133 @@ The API returns the following response:
 // TESTRESPONSE[s/"version": <version>/"version": $body.snapshots.0.version/]
 // TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/]
 // TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/]
-// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.876Z"/"end_time": $body.snapshots.0.end_time/]
+// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
+// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
+// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]
+
+The following request returns information for all snapshots with prefix `snapshot` in the `my_repository` repository,
+limiting the response size to 2 and sorting by snapshot name.
+
+[source,console]
+----
+GET /_snapshot/my_repository/snapshot*?size=2&sort=name
+----
+
+The API returns the following response:
+
+[source,console-result]
+----
+{
+  "snapshots": [
+    {
+      "snapshot": "snapshot_1",
+      "uuid": "dKb54xw67gvdRctLCxSket",
+      "repository": "my_repository",
+      "version_id": <version_id>,
+      "version": <version>,
+      "indices": [],
+      "data_streams": [],
+      "feature_states": [],
+      "include_global_state": true,
+      "state": "SUCCESS",
+      "start_time": "2020-07-06T21:55:18.129Z",
+      "start_time_in_millis": 1593093628850,
+      "end_time": "2020-07-06T21:55:18.129Z",
+      "end_time_in_millis": 1593094752018,
+      "duration_in_millis": 0,
+      "failures": [],
+      "shards": {
+        "total": 0,
+        "failed": 0,
+        "successful": 0
+      }
+    },
+    {
+      "snapshot": "snapshot_2",
+      "uuid": "vdRctLCxSketdKb54xw67g",
+      "repository": "my_repository",
+      "version_id": <version_id>,
+      "version": <version>,
+      "indices": [],
+      "data_streams": [],
+      "feature_states": [],
+      "include_global_state": true,
+      "state": "SUCCESS",
+      "start_time": "2020-07-06T21:55:18.130Z",
+      "start_time_in_millis": 1593093628851,
+      "end_time": "2020-07-06T21:55:18.130Z",
+      "end_time_in_millis": 1593094752019,
+      "duration_in_millis": 1,
+      "failures": [],
+      "shards": {
+        "total": 0,
+        "failed": 0,
+        "successful": 0
+      }
+    }
+  ],
+  "next": "c25hcHNob3RfMixteV9yZXBvc2l0b3J5LHNuYXBzaG90XzI="
+}
+----
+// TESTRESPONSE[s/"uuid": "dKb54xw67gvdRctLCxSket"/"uuid": $body.snapshots.0.uuid/]
+// TESTRESPONSE[s/"uuid": "vdRctLCxSketdKb54xw67g"/"uuid": $body.snapshots.1.uuid/]
+// TESTRESPONSE[s/"version_id": <version_id>/"version_id": $body.snapshots.0.version_id/]
+// TESTRESPONSE[s/"version": <version>/"version": $body.snapshots.0.version/]
+// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/]
+// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.130Z"/"start_time": $body.snapshots.1.start_time/]
+// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/]
+// TESTRESPONSE[s/"start_time_in_millis": 1593093628851/"start_time_in_millis": $body.snapshots.1.start_time_in_millis/]
+// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
+// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.130Z"/"end_time": $body.snapshots.1.end_time/]
 // TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
+// TESTRESPONSE[s/"end_time_in_millis": 1593094752019/"end_time_in_millis": $body.snapshots.1.end_time_in_millis/]
 // TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]
+// TESTRESPONSE[s/"duration_in_millis": 1/"duration_in_millis": $body.snapshots.1.duration_in_millis/]
+
+A subsequent request for the remaining snapshots can then be made using the `next` value from the previous response as `after` parameter.
+
+[source,console]
+----
+GET /_snapshot/my_repository/snapshot*?size=2&sort=name&after=c25hcHNob3RfMixteV9yZXBvc2l0b3J5LHNuYXBzaG90XzI=
+----
+
+The API returns the following response:
+
+[source,console-result]
+----
+{
+  "snapshots": [
+    {
+      "snapshot": "snapshot_3",
+      "uuid": "dRctdKb54xw67gvLCxSket",
+      "repository": "my_repository",
+      "version_id": <version_id>,
+      "version": <version>,
+      "indices": [],
+      "data_streams": [],
+      "feature_states": [],
+      "include_global_state": true,
+      "state": "SUCCESS",
+      "start_time": "2020-07-06T21:55:18.129Z",
+      "start_time_in_millis": 1593093628850,
+      "end_time": "2020-07-06T21:55:18.129Z",
+      "end_time_in_millis": 1593094752018,
+      "duration_in_millis": 0,
+      "failures": [],
+      "shards": {
+        "total": 0,
+        "failed": 0,
+        "successful": 0
+      }
+    }
+  ]
+}
+----
+// TESTRESPONSE[s/"uuid": "dRctdKb54xw67gvLCxSket"/"uuid": $body.snapshots.0.uuid/]
+// TESTRESPONSE[s/"version_id": <version_id>/"version_id": $body.snapshots.0.version_id/]
+// TESTRESPONSE[s/"version": <version>/"version": $body.snapshots.0.version/]
+// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/]
+// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/]
+// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
+// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
+// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]