Skip to content

Commit 680d6ed

Browse files
andrershovandrershov
authored
Get snapshots support for multiple repositories (#42090)
This commit adds multiple repositories support to get snapshots request. If some repository throws an exception this method does not fail fast instead, it returns results for all repositories. This PR is opened in favour of #41799, because we decided to change the response format in a non-BwC manner. It makes sense to read a discussion of the aforementioned PR. This is the continuation of work done here #15151.
1 parent 43ec1d8 commit 680d6ed

File tree

51 files changed

+972
-446
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+972
-446
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throw
108108

109109
static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) {
110110
RequestConverters.EndpointBuilder endpointBuilder = new RequestConverters.EndpointBuilder().addPathPartAsIs("_snapshot")
111-
.addPathPart(getSnapshotsRequest.repository());
111+
.addCommaSeparatedPathParts(getSnapshotsRequest.repositories());
112112
String endpoint;
113113
if (getSnapshotsRequest.snapshots().length == 0) {
114114
endpoint = endpointBuilder.addPathPart("_all").build();

‎client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java‎

Lines changed: 43 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,16 @@
4141
import org.elasticsearch.repositories.fs.FsRepository;
4242
import org.elasticsearch.rest.RestStatus;
4343
import org.elasticsearch.snapshots.RestoreInfo;
44-
import org.elasticsearch.snapshots.SnapshotInfo;
44+
import org.mockito.internal.util.collections.Sets;
4545

4646
import java.io.IOException;
4747
import java.util.Collections;
4848
import java.util.HashMap;
4949
import java.util.Map;
50-
import java.util.stream.Collectors;
5150

52-
import static org.hamcrest.Matchers.contains;
5351
import static org.hamcrest.Matchers.equalTo;
5452
import static org.hamcrest.Matchers.greaterThan;
53+
import static org.hamcrest.Matchers.hasSize;
5554
import static org.hamcrest.Matchers.is;
5655

5756
public class SnapshotIT extends ESRestHighLevelClientTestCase {
@@ -61,14 +60,14 @@ private AcknowledgedResponse createTestRepository(String repository, String type
6160
request.settings(settings, XContentType.JSON);
6261
request.type(type);
6362
return execute(request, highLevelClient().snapshot()::createRepository,
64-
highLevelClient().snapshot()::createRepositoryAsync);
63+
highLevelClient().snapshot()::createRepositoryAsync);
6564
}
6665

6766
private CreateSnapshotResponse createTestSnapshot(CreateSnapshotRequest createSnapshotRequest) throws IOException {
6867
// assumes the repository already exists
6968

7069
return execute(createSnapshotRequest, highLevelClient().snapshot()::create,
71-
highLevelClient().snapshot()::createAsync);
70+
highLevelClient().snapshot()::createAsync);
7271
}
7372

7473
public void testCreateRepository() throws IOException {
@@ -84,7 +83,7 @@ public void testSnapshotGetRepositoriesUsingParams() throws IOException {
8483
GetRepositoriesRequest request = new GetRepositoriesRequest();
8584
request.repositories(new String[]{testRepository});
8685
GetRepositoriesResponse response = execute(request, highLevelClient().snapshot()::getRepository,
87-
highLevelClient().snapshot()::getRepositoryAsync);
86+
highLevelClient().snapshot()::getRepositoryAsync);
8887
assertThat(1, equalTo(response.repositories().size()));
8988
}
9089

@@ -93,19 +92,19 @@ public void testSnapshotGetDefaultRepositories() throws IOException {
9392
assertTrue(createTestRepository("test", FsRepository.TYPE, "{\"location\": \".\"}").isAcknowledged());
9493

9594
GetRepositoriesResponse response = execute(new GetRepositoriesRequest(), highLevelClient().snapshot()::getRepository,
96-
highLevelClient().snapshot()::getRepositoryAsync);
95+
highLevelClient().snapshot()::getRepositoryAsync);
9796
assertThat(2, equalTo(response.repositories().size()));
9897
}
9998

10099
public void testSnapshotGetRepositoriesNonExistent() {
101100
String repository = "doesnotexist";
102101
GetRepositoriesRequest request = new GetRepositoriesRequest(new String[]{repository});
103102
ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(request,
104-
highLevelClient().snapshot()::getRepository, highLevelClient().snapshot()::getRepositoryAsync));
103+
highLevelClient().snapshot()::getRepository, highLevelClient().snapshot()::getRepositoryAsync));
105104

106105
assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND));
107106
assertThat(exception.getMessage(), equalTo(
108-
"Elasticsearch exception [type=repository_missing_exception, reason=[" + repository + "] missing]"));
107+
"Elasticsearch exception [type=repository_missing_exception, reason=[" + repository + "] missing]"));
109108
}
110109

111110
public void testSnapshotDeleteRepository() throws IOException {
@@ -114,12 +113,12 @@ public void testSnapshotDeleteRepository() throws IOException {
114113

115114
GetRepositoriesRequest request = new GetRepositoriesRequest();
116115
GetRepositoriesResponse response = execute(request, highLevelClient().snapshot()::getRepository,
117-
highLevelClient().snapshot()::getRepositoryAsync);
116+
highLevelClient().snapshot()::getRepositoryAsync);
118117
assertThat(1, equalTo(response.repositories().size()));
119118

120119
DeleteRepositoryRequest deleteRequest = new DeleteRepositoryRequest(repository);
121120
AcknowledgedResponse deleteResponse = execute(deleteRequest, highLevelClient().snapshot()::deleteRepository,
122-
highLevelClient().snapshot()::deleteRepositoryAsync);
121+
highLevelClient().snapshot()::deleteRepositoryAsync);
123122

124123
assertTrue(deleteResponse.isAcknowledged());
125124
}
@@ -130,7 +129,7 @@ public void testVerifyRepository() throws IOException {
130129

131130
VerifyRepositoryRequest request = new VerifyRepositoryRequest("test");
132131
VerifyRepositoryResponse response = execute(request, highLevelClient().snapshot()::verifyRepository,
133-
highLevelClient().snapshot()::verifyRepositoryAsync);
132+
highLevelClient().snapshot()::verifyRepositoryAsync);
134133
assertThat(response.getNodes().size(), equalTo(1));
135134
}
136135

@@ -153,25 +152,31 @@ public void testCreateSnapshot() throws IOException {
153152
if (waitForCompletion == false) {
154153
// If we don't wait for the snapshot to complete we have to cancel it to not leak the snapshot task
155154
AcknowledgedResponse deleteResponse = execute(
156-
new DeleteSnapshotRequest(repository, snapshot),
157-
highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync
155+
new DeleteSnapshotRequest(repository, snapshot),
156+
highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync
158157
);
159158
assertTrue(deleteResponse.isAcknowledged());
160159
}
161160
}
162161

163162
public void testGetSnapshots() throws IOException {
164-
String repository = "test_repository";
163+
String repository1 = "test_repository1";
164+
String repository2 = "test_repository2";
165165
String snapshot1 = "test_snapshot1";
166166
String snapshot2 = "test_snapshot2";
167167

168-
AcknowledgedResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}");
168+
AcknowledgedResponse putRepositoryResponse =
169+
createTestRepository(repository1, FsRepository.TYPE, "{\"location\": \"loc1\"}");
169170
assertTrue(putRepositoryResponse.isAcknowledged());
170171

171-
CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository, snapshot1);
172+
AcknowledgedResponse putRepositoryResponse2 =
173+
createTestRepository(repository2, FsRepository.TYPE, "{\"location\": \"loc2\"}");
174+
assertTrue(putRepositoryResponse2.isAcknowledged());
175+
176+
CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository1, snapshot1);
172177
createSnapshotRequest1.waitForCompletion(true);
173178
CreateSnapshotResponse putSnapshotResponse1 = createTestSnapshot(createSnapshotRequest1);
174-
CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository, snapshot2);
179+
CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository2, snapshot2);
175180
createSnapshotRequest2.waitForCompletion(true);
176181
Map<String, Object> originalMetadata = randomUserMetadata();
177182
createSnapshotRequest2.userMetadata(originalMetadata);
@@ -180,28 +185,26 @@ public void testGetSnapshots() throws IOException {
180185
assertEquals(RestStatus.OK, putSnapshotResponse1.status());
181186
assertEquals(RestStatus.OK, putSnapshotResponse2.status());
182187

183-
GetSnapshotsRequest request;
184-
if (randomBoolean()) {
185-
request = new GetSnapshotsRequest(repository);
186-
} else if (randomBoolean()) {
187-
request = new GetSnapshotsRequest(repository, new String[] {"_all"});
188+
GetSnapshotsRequest request = new GetSnapshotsRequest(
189+
randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{repository1, repository2}),
190+
randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{snapshot1, snapshot2})
191+
);
192+
request.ignoreUnavailable(true);
188193

189-
} else {
190-
request = new GetSnapshotsRequest(repository, new String[] {snapshot1, snapshot2});
191-
}
192194
GetSnapshotsResponse response = execute(request, highLevelClient().snapshot()::get, highLevelClient().snapshot()::getAsync);
193195

194-
assertEquals(2, response.getSnapshots().size());
195-
assertThat(response.getSnapshots().stream().map((s) -> s.snapshotId().getName()).collect(Collectors.toList()),
196-
contains("test_snapshot1", "test_snapshot2"));
197-
response.getSnapshots().stream()
198-
.filter(s -> s.snapshotId().getName().equals("test_snapshot2"))
199-
.findFirst()
200-
.map(SnapshotInfo::userMetadata)
201-
.ifPresentOrElse(metadata -> assertEquals(originalMetadata, metadata),
202-
() -> assertNull("retrieved metadata is null, expected non-null metadata", originalMetadata));
196+
assertThat(response.isFailed(), is(false));
197+
assertThat(response.getRepositories(), equalTo(Sets.newSet(repository1, repository2)));
198+
199+
assertThat(response.getSnapshots(repository1), hasSize(1));
200+
assertThat(response.getSnapshots(repository1).get(0).snapshotId().getName(), equalTo(snapshot1));
201+
202+
assertThat(response.getSnapshots(repository2), hasSize(1));
203+
assertThat(response.getSnapshots(repository2).get(0).snapshotId().getName(), equalTo(snapshot2));
204+
assertThat(response.getSnapshots(repository2).get(0).userMetadata(), equalTo(originalMetadata));
203205
}
204206

207+
205208
public void testSnapshotsStatus() throws IOException {
206209
String testRepository = "test";
207210
String testSnapshot = "snapshot";
@@ -223,7 +226,7 @@ public void testSnapshotsStatus() throws IOException {
223226
request.repository(testRepository);
224227
request.snapshots(new String[]{testSnapshot});
225228
SnapshotsStatusResponse response = execute(request, highLevelClient().snapshot()::status,
226-
highLevelClient().snapshot()::statusAsync);
229+
highLevelClient().snapshot()::statusAsync);
227230
assertThat(response.getSnapshots().size(), equalTo(1));
228231
assertThat(response.getSnapshots().get(0).getSnapshot().getRepository(), equalTo(testRepository));
229232
assertThat(response.getSnapshots().get(0).getSnapshot().getSnapshotId().getName(), equalTo(testSnapshot));
@@ -260,7 +263,7 @@ public void testRestoreSnapshot() throws IOException {
260263
request.renameReplacement(restoredIndex);
261264

262265
RestoreSnapshotResponse response = execute(request, highLevelClient().snapshot()::restore,
263-
highLevelClient().snapshot()::restoreAsync);
266+
highLevelClient().snapshot()::restoreAsync);
264267

265268
RestoreInfo restoreInfo = response.getRestoreInfo();
266269
assertThat(restoreInfo.name(), equalTo(testSnapshot));
@@ -301,17 +304,17 @@ private static Map<String, Object> randomUserMetadata() {
301304
for (int i = 0; i < fields; i++) {
302305
if (randomBoolean()) {
303306
metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)),
304-
randomAlphaOfLengthBetween(5, 5));
307+
randomAlphaOfLengthBetween(5, 5));
305308
} else {
306309
Map<String, Object> nested = new HashMap<>();
307310
long nestedFields = randomLongBetween(0, 4);
308311
for (int j = 0; j < nestedFields; j++) {
309312
nested.put(randomValueOtherThanMany(nested::containsKey, () -> randomAlphaOfLengthBetween(2,10)),
310-
randomAlphaOfLengthBetween(5, 5));
313+
randomAlphaOfLengthBetween(5, 5));
311314
}
312315
metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)), nested);
313316
}
314317
}
315318
return metadata;
316319
}
317-
}
320+
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@
4141

4242
import java.io.IOException;
4343
import java.nio.file.Path;
44-
import java.util.Arrays;
4544
import java.util.HashMap;
4645
import java.util.Locale;
4746
import java.util.Map;
@@ -148,15 +147,16 @@ public void testCreateSnapshot() throws IOException {
148147

149148
public void testGetSnapshots() {
150149
Map<String, String> expectedParams = new HashMap<>();
151-
String repository = RequestConvertersTests.randomIndicesNames(1, 1)[0];
150+
String repository1 = randomAlphaOfLength(10);
151+
String repository2 = randomAlphaOfLength(10);
152152
String snapshot1 = "snapshot1-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT);
153153
String snapshot2 = "snapshot2-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT);
154154

155-
String endpoint = String.format(Locale.ROOT, "/_snapshot/%s/%s,%s", repository, snapshot1, snapshot2);
155+
String endpoint = String.format(Locale.ROOT, "/_snapshot/%s,%s/%s,%s", repository1, repository2, snapshot1, snapshot2);
156156

157157
GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest();
158-
getSnapshotsRequest.repository(repository);
159-
getSnapshotsRequest.snapshots(Arrays.asList(snapshot1, snapshot2).toArray(new String[0]));
158+
getSnapshotsRequest.repositories(repository1, repository2);
159+
getSnapshotsRequest.snapshots(new String[]{snapshot1, snapshot2});
160160
RequestConvertersTests.setRandomMasterTimeout(getSnapshotsRequest, expectedParams);
161161

162162
if (randomBoolean()) {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -590,7 +590,7 @@ public void testSnapshotGetSnapshots() throws IOException {
590590
// end::get-snapshots-request
591591

592592
// tag::get-snapshots-request-repositoryName
593-
request.repository(repositoryName); // <1>
593+
request.repositories(repositoryName); // <1>
594594
// end::get-snapshots-request-repositoryName
595595

596596
// tag::get-snapshots-request-snapshots
@@ -616,7 +616,7 @@ public void testSnapshotGetSnapshots() throws IOException {
616616
// end::get-snapshots-execute
617617

618618
// tag::get-snapshots-response
619-
List<SnapshotInfo> snapshotsInfos = response.getSnapshots();
619+
List<SnapshotInfo> snapshotsInfos = response.getSnapshots(repositoryName);
620620
SnapshotInfo snapshotInfo = snapshotsInfos.get(0);
621621
RestStatus restStatus = snapshotInfo.status(); // <1>
622622
SnapshotId snapshotId = snapshotInfo.snapshotId(); // <2>

‎docs/reference/cat/snapshots.asciidoc‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
[[cat-snapshots]]
22
== cat snapshots
33

4-
The `snapshots` command shows all snapshots that belong to a specific repository.
4+
The `snapshots` command shows all snapshots that belong to a specific repository
5+
or multiple repositories.
56
To find a list of available repositories to query, the command `/_cat/repositories` can be used.
67
Querying the snapshots of a repository named `repo1` then looks as follows.
78

@@ -18,9 +19,9 @@ Which looks like:
1819

1920
[source,txt]
2021
--------------------------------------------------
21-
id status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards
22-
snap1 FAILED 1445616705 18:11:45 1445616978 18:16:18 4.6m 1 4 1 5
23-
snap2 SUCCESS 1445634298 23:04:58 1445634672 23:11:12 6.2m 2 10 0 10
22+
id repository status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards
23+
snap1 repo1 FAILED 1445616705 18:11:45 1445616978 18:16:18 4.6m 1 4 1 5
24+
snap2 repo1 SUCCESS 1445634298 23:04:58 1445634672 23:11:12 6.2m 2 10 0 10
2425
--------------------------------------------------
2526
// TESTRESPONSE[s/FAILED/SUCCESS/ s/14456\d+/\\d+/ s/\d+(\.\d+)?(m|s|ms)/\\d+(\\.\\d+)?(m|s|ms)/]
2627
// TESTRESPONSE[s/\d+:\d+:\d+/\\d+:\\d+:\\d+/]
@@ -32,3 +33,17 @@ Each snapshot contains information about when it was started and stopped.
3233
Start and stop timestamps are available in two formats.
3334
The `HH:MM:SS` output is simply for quick human consumption.
3435
The epoch time retains more information, including date, and is machine sortable if the snapshot process spans days.
36+
37+
It is also possible to get the list of snapshots from multiple repositories.
38+
Here are some examples:
39+
40+
[source,js]
41+
--------------------------------------------------
42+
GET /_cat/snapshots/_all
43+
GET /_cat/snapshots/repo1,repo2
44+
GET /_cat/snapshots/repo*
45+
--------------------------------------------------
46+
// CONSOLE
47+
// TEST[skip:no repo2]
48+
49+
Please note that if one of the repositories fails during the request you will get an exception instead of the table.

‎docs/reference/migration/migrate_8_0/snapshots.asciidoc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@
99

1010
// end::notable-breaking-changes[]
1111

12+
[float]
13+
=== Get snapshots response format is changed
14+
It's possible to get snapshots from multiple repositories in one go. The response format has changed
15+
and now contains separate response for each repository. See <<modules-snapshots>> for more information.
16+
1217
[float]
1318
==== Deprecated node level compress setting removed
1419

‎docs/reference/modules/snapshots.asciidoc‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,16 @@ that setting `verbose` to `false` will omit all other information about the snap
472472
such as status information, the number of snapshotted shards, etc. The default
473473
value of the `verbose` parameter is `true`.
474474

475+
It is also possible to retrieve snapshots from multiple repositories in one go, for example:
476+
[source,sh]
477+
-----------------------------------
478+
GET /_snapshot/_all
479+
GET /_snapshot/my_backup,my_fs_backup
480+
GET /_snapshot/my*/snap*
481+
-----------------------------------
482+
// CONSOLE
483+
// TEST[skip:no my_fs_backup]
484+
475485
A currently running snapshot can be retrieved using the following command:
476486

477487
[source,sh]

‎modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public void testUrlRepository() throws Exception {
8888
.prepareGetSnapshots("test-repo")
8989
.setSnapshots("test-snap")
9090
.get()
91-
.getSnapshots()
91+
.getSnapshots("test-repo")
9292
.get(0)
9393
.state();
9494
assertThat(state, equalTo(SnapshotState.SUCCESS));
@@ -116,16 +116,16 @@ public void testUrlRepository() throws Exception {
116116

117117
logger.info("--> list available shapshots");
118118
GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
119-
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
120-
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
119+
assertThat(getSnapshotsResponse.getSnapshots("url-repo"), notNullValue());
120+
assertThat(getSnapshotsResponse.getSnapshots("url-repo").size(), equalTo(1));
121121

122122
logger.info("--> delete snapshot");
123123
AcknowledgedResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
124124
assertAcked(deleteSnapshotResponse);
125125

126126
logger.info("--> list available shapshot again, no snapshots should be returned");
127127
getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
128-
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
129-
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
128+
assertThat(getSnapshotsResponse.getSnapshots("url-repo"), notNullValue());
129+
assertThat(getSnapshotsResponse.getSnapshots("url-repo").size(), equalTo(0));
130130
}
131131
}

0 commit comments

Comments
 (0)