Skip to content

Commit 6eba38a

Browse files
imotovimotov
committed
Improve deletion of corrupted snapshots
Makes it possible to delete snapshots that are missing some of the metadata files. This can happen if snapshot creation failed because repository drive ran out of disk space. Closes #6383
1 parent 2033cdb commit 6eba38a

File tree

2 files changed

+91
-60
lines changed

2 files changed

+91
-60
lines changed

‎src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java‎

Lines changed: 52 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ public void initializeSnapshot(SnapshotId snapshotId, ImmutableList<String> indi
259259
@Override
260260
public void deleteSnapshot(SnapshotId snapshotId) {
261261
Snapshot snapshot = readSnapshot(snapshotId);
262-
MetaData metaData = readSnapshotMetaData(snapshotId, snapshot.indices());
262+
MetaData metaData = readSnapshotMetaData(snapshotId, snapshot.indices(), true);
263263
try {
264264
String blobName = snapshotBlobName(snapshotId);
265265
// Delete snapshot file first so we wouldn't end up with partially deleted snapshot that looks OK
@@ -284,11 +284,13 @@ public void deleteSnapshot(SnapshotId snapshotId) {
284284
try {
285285
indexMetaDataBlobContainer.deleteBlob(blobName);
286286
} catch (IOException ex) {
287-
throw new SnapshotException(snapshotId, "failed to delete metadata", ex);
287+
logger.warn("[{}] failed to delete metadata for index [{}]", ex, snapshotId, index);
288288
}
289289
IndexMetaData indexMetaData = metaData.index(index);
290-
for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) {
291-
indexShardRepository.delete(snapshotId, new ShardId(index, i));
290+
if (indexMetaData != null) {
291+
for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) {
292+
indexShardRepository.delete(snapshotId, new ShardId(index, i));
293+
}
292294
}
293295
}
294296
} catch (IOException ex) {
@@ -367,41 +369,7 @@ public ImmutableList<SnapshotId> snapshots() {
367369
*/
368370
@Override
369371
public MetaData readSnapshotMetaData(SnapshotId snapshotId, ImmutableList<String> indices) {
370-
MetaData metaData;
371-
try {
372-
byte[] data = snapshotsBlobContainer.readBlobFully(metaDataBlobName(snapshotId));
373-
metaData = readMetaData(data);
374-
} catch (FileNotFoundException | NoSuchFileException ex) {
375-
throw new SnapshotMissingException(snapshotId, ex);
376-
} catch (IOException ex) {
377-
throw new SnapshotException(snapshotId, "failed to get snapshots", ex);
378-
}
379-
MetaData.Builder metaDataBuilder = MetaData.builder(metaData);
380-
for (String index : indices) {
381-
BlobPath indexPath = basePath().add("indices").add(index);
382-
ImmutableBlobContainer indexMetaDataBlobContainer = blobStore().immutableBlobContainer(indexPath);
383-
XContentParser parser = null;
384-
try {
385-
byte[] data = indexMetaDataBlobContainer.readBlobFully(snapshotBlobName(snapshotId));
386-
parser = XContentHelper.createParser(data, 0, data.length);
387-
XContentParser.Token token;
388-
if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) {
389-
IndexMetaData indexMetaData = IndexMetaData.Builder.fromXContent(parser);
390-
if ((token = parser.nextToken()) == XContentParser.Token.END_OBJECT) {
391-
metaDataBuilder.put(indexMetaData, false);
392-
continue;
393-
}
394-
}
395-
throw new ElasticsearchParseException("unexpected token [" + token + "]");
396-
} catch (IOException ex) {
397-
throw new SnapshotException(snapshotId, "failed to read metadata", ex);
398-
} finally {
399-
if (parser != null) {
400-
parser.close();
401-
}
402-
}
403-
}
404-
return metaDataBuilder.build();
372+
return readSnapshotMetaData(snapshotId, indices, false);
405373
}
406374

407375
/**
@@ -439,6 +407,48 @@ public Snapshot readSnapshot(SnapshotId snapshotId) {
439407
}
440408
}
441409

410+
private MetaData readSnapshotMetaData(SnapshotId snapshotId, ImmutableList<String> indices, boolean ignoreIndexErrors) {
411+
MetaData metaData;
412+
try {
413+
byte[] data = snapshotsBlobContainer.readBlobFully(metaDataBlobName(snapshotId));
414+
metaData = readMetaData(data);
415+
} catch (FileNotFoundException | NoSuchFileException ex) {
416+
throw new SnapshotMissingException(snapshotId, ex);
417+
} catch (IOException ex) {
418+
throw new SnapshotException(snapshotId, "failed to get snapshots", ex);
419+
}
420+
MetaData.Builder metaDataBuilder = MetaData.builder(metaData);
421+
for (String index : indices) {
422+
BlobPath indexPath = basePath().add("indices").add(index);
423+
ImmutableBlobContainer indexMetaDataBlobContainer = blobStore().immutableBlobContainer(indexPath);
424+
try {
425+
byte[] data = indexMetaDataBlobContainer.readBlobFully(snapshotBlobName(snapshotId));
426+
try (XContentParser parser = XContentHelper.createParser(data, 0, data.length)) {
427+
XContentParser.Token token;
428+
if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) {
429+
IndexMetaData indexMetaData = IndexMetaData.Builder.fromXContent(parser);
430+
if ((token = parser.nextToken()) == XContentParser.Token.END_OBJECT) {
431+
metaDataBuilder.put(indexMetaData, false);
432+
continue;
433+
}
434+
}
435+
if (!ignoreIndexErrors) {
436+
throw new ElasticsearchParseException("unexpected token [" + token + "]");
437+
} else {
438+
logger.warn("[{}] [{}] unexpected token while reading snapshot metadata [{}]", snapshotId, index, token);
439+
}
440+
}
441+
} catch (IOException ex) {
442+
if (!ignoreIndexErrors) {
443+
throw new SnapshotException(snapshotId, "failed to read metadata", ex);
444+
} else {
445+
logger.warn("[{}] [{}] failed to read metadata for index", snapshotId, index, ex);
446+
}
447+
}
448+
}
449+
return metaDataBuilder.build();
450+
}
451+
442452
/**
443453
* Configures RateLimiter based on repository and global settings
444454
*
@@ -465,9 +475,7 @@ private RateLimiter getRateLimiter(RepositorySettings repositorySettings, String
465475
* @throws IOException parse exceptions
466476
*/
467477
private BlobStoreSnapshot readSnapshot(byte[] data) throws IOException {
468-
XContentParser parser = null;
469-
try {
470-
parser = XContentHelper.createParser(data, 0, data.length);
478+
try (XContentParser parser = XContentHelper.createParser(data, 0, data.length)) {
471479
XContentParser.Token token;
472480
if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) {
473481
if ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME) {
@@ -479,10 +487,6 @@ private BlobStoreSnapshot readSnapshot(byte[] data) throws IOException {
479487
}
480488
}
481489
throw new ElasticsearchParseException("unexpected token [" + token + "]");
482-
} finally {
483-
if (parser != null) {
484-
parser.close();
485-
}
486490
}
487491
}
488492

@@ -494,9 +498,7 @@ private BlobStoreSnapshot readSnapshot(byte[] data) throws IOException {
494498
* @throws IOException parse exceptions
495499
*/
496500
private MetaData readMetaData(byte[] data) throws IOException {
497-
XContentParser parser = null;
498-
try {
499-
parser = XContentHelper.createParser(data, 0, data.length);
501+
try (XContentParser parser = XContentHelper.createParser(data, 0, data.length)) {
500502
XContentParser.Token token;
501503
if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) {
502504
if ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME) {
@@ -508,10 +510,6 @@ private MetaData readMetaData(byte[] data) throws IOException {
508510
}
509511
}
510512
throw new ElasticsearchParseException("unexpected token [" + token + "]");
511-
} finally {
512-
if (parser != null) {
513-
parser.close();
514-
}
515513
}
516514
}
517515

@@ -615,9 +613,7 @@ protected void writeSnapshotList(ImmutableList<SnapshotId> snapshots) throws IOE
615613
protected ImmutableList<SnapshotId> readSnapshotList() throws IOException {
616614
byte[] data = snapshotsBlobContainer.readBlobFully(SNAPSHOTS_FILE);
617615
ArrayList<SnapshotId> snapshots = new ArrayList<>();
618-
XContentParser parser = null;
619-
try {
620-
parser = XContentHelper.createParser(data, 0, data.length);
616+
try (XContentParser parser = XContentHelper.createParser(data, 0, data.length)) {
621617
if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
622618
if (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
623619
String currentFieldName = parser.currentName();
@@ -630,10 +626,6 @@ protected ImmutableList<SnapshotId> readSnapshotList() throws IOException {
630626
}
631627
}
632628
}
633-
} finally {
634-
if (parser != null) {
635-
parser.close();
636-
}
637629
}
638630
return ImmutableList.copyOf(snapshots);
639631
}

‎src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,45 @@ public void deleteSnapshotTest() throws Exception {
629629
assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0]));
630630
}
631631

632+
@Test
633+
public void deleteSnapshotWithMissingIndexAndShardMetadataTest() throws Exception {
634+
Client client = client();
635+
636+
File repo = newTempDir(LifecycleScope.SUITE);
637+
logger.info("--> creating repository at " + repo.getAbsolutePath());
638+
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
639+
.setType("fs").setSettings(ImmutableSettings.settingsBuilder()
640+
.put("location", repo)
641+
.put("compress", false)
642+
.put("chunk_size", randomIntBetween(100, 1000))));
643+
644+
createIndex("test-idx-1", "test-idx-2");
645+
ensureYellow();
646+
logger.info("--> indexing some data");
647+
indexRandom(true,
648+
client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"),
649+
client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar"));
650+
651+
logger.info("--> creating snapshot");
652+
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get();
653+
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
654+
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
655+
656+
logger.info("--> delete index metadata and shard metadata");
657+
File indices = new File(repo, "indices");
658+
File testIndex1 = new File(indices, "test-idx-1");
659+
File testIndex2 = new File(indices, "test-idx-2");
660+
File testIndex2Shard0 = new File(testIndex2, "0");
661+
new File(testIndex1, "snapshot-test-snap-1").delete();
662+
new File(testIndex2Shard0, "snapshot-test-snap-1").delete();
663+
664+
logger.info("--> delete snapshot");
665+
client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get();
666+
667+
logger.info("--> make sure snapshot doesn't exist");
668+
assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class);
669+
}
670+
632671
@Test
633672
@TestLogging("snapshots:TRACE")
634673
public void snapshotClosedIndexTest() throws Exception {

0 commit comments

Comments
 (0)