diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 84b628d1ecc2f..9f38bcd858c47 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -771,7 +771,7 @@ private static void ensureNoSearchableSnapshotsIndicesInUse(ClusterState cluster } } - private static boolean indexSettingsMatchRepositoryMetadata(IndexMetadata indexMetadata, RepositoryMetadata repositoryMetadata) { + public static boolean indexSettingsMatchRepositoryMetadata(IndexMetadata indexMetadata, RepositoryMetadata repositoryMetadata) { if (indexMetadata.isSearchableSnapshot()) { final Settings indexSettings = indexMetadata.getSettings(); final String indexRepositoryUuid = indexSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 6eef30f41f27c..0f95cc53af53d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -47,6 +47,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; +import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -114,6 +115,7 @@ import static java.util.Collections.unmodifiableList; import static org.elasticsearch.cluster.SnapshotsInProgress.completed; +import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY; /** * Service responsible for creating snapshots. This service runs all the steps executed on the master node during snapshot creation and @@ -2158,6 +2160,11 @@ public ClusterState execute(ClusterState currentState) { ); } } + + for (SnapshotId snapshotId : snapshotIds) { + ensureNoSearchableSnapshotsIndicesInUse(repository.getMetadata(), snapshotId, currentState); + } + // Snapshot ids that will have to be physically deleted from the repository final Set snapshotIdsRequiringCleanup = new HashSet<>(snapshotIds); final SnapshotsInProgress updatedSnapshots = snapshotsInProgress.withUpdatedEntriesForRepo( @@ -2956,6 +2963,35 @@ static Map filterDataStreamAliases( .collect(Collectors.toMap(DataStreamAlias::getName, Function.identity())); } + private static void ensureNoSearchableSnapshotsIndicesInUse(RepositoryMetadata repository, SnapshotId snapshot, ClusterState state) { + long count = 0L; + List indices = null; + for (IndexMetadata indexMetadata : state.metadata()) { + final Settings indexSettings = indexMetadata.getSettings(); + if (RepositoriesService.indexSettingsMatchRepositoryMetadata(indexMetadata, repository) + && Objects.equals(snapshot.getUUID(), indexSettings.get(SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY))) { + if (indices == null) { + indices = new ArrayList<>(); + } + if (indices.size() < 5) { + indices.add(indexMetadata.getIndex()); + } + count += 1L; + } + } + if (indices != null && indices.isEmpty() == false) { + throw new SnapshotException( + repository.name(), + snapshot.toString(), + "found " + + count + + " searchable snapshots indices that use the snapshot: " + + Strings.collectionToCommaDelimitedString(indices) + + (count > indices.size() ? ",..." : "") + ); + } + } + /** * Adds snapshot completion listener * diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/repository.yml b/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/repository.yml index 8c43c0682c33b..212e281e1904e 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/repository.yml +++ b/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/repository.yml @@ -54,7 +54,7 @@ setup: --- "Delete repository when a snapshot is mounted as an index": - skip: - version: " - 7.99.99" + version: " - 7.13.99" reason: "starting 8.0, an error occurs when a repository used by snapshot backed indices is deleted" - do: @@ -93,3 +93,48 @@ setup: snapshot.delete_repository: repository: repository-fs + +--- +"Delete snapshot when the snapshot is mounted as an index": + - skip: + version: " - 7.13.99" + reason: "starting 8.0, an error occurs when a snapshot used by snapshot backed indices is deleted" + + - do: + searchable_snapshots.mount: + repository: repository-fs + snapshot: snapshot + wait_for_completion: true + body: + index: docs + renamed_index: mounted-docs + + - match: { snapshot.snapshot: snapshot } + - match: { snapshot.shards.failed: 0 } + - match: { snapshot.shards.successful: 1 } + + # Returns an illegal state exception + - do: + catch: request + snapshot.delete: + repository: repository-fs + snapshot: snapshot + + - do: + search: + index: mounted-docs + body: + query: + match_all: { } + + - match: { hits.total.value: 3 } + + - do: + indices.delete: + index: mounted-docs + + - do: + snapshot.delete: + repository: repository-fs + snapshot: snapshot + diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java index 6dfb07e2db852..a97393e2740bd 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.searchablesnapshots; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -16,12 +17,17 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.repositories.RepositoryConflictException; import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.snapshots.SnapshotException; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotRestoreException; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; @@ -34,6 +40,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class SearchableSnapshotsRepositoryIntegTests extends BaseFrozenSearchableSnapshotsIntegTestCase { @@ -60,8 +68,7 @@ public void testRepositoryUsedBySearchableSnapshotCanBeUpdatedButNotUnregistered createSnapshot(repositoryName, snapshotName, List.of(indexName)); assertAcked(client().admin().indices().prepareDelete(indexName)); - final int nbMountedIndices = 1; - randomIntBetween(1, 5); + final int nbMountedIndices = randomIntBetween(1, 5); final String[] mountedIndices = new String[nbMountedIndices]; for (int i = 0; i < nbMountedIndices; i++) { @@ -402,6 +409,116 @@ public void testRestoreSearchableSnapshotIndexWithDifferentSettingsConflicts() t assertAcked(client().admin().indices().prepareDelete("restored-with-same-setting-*")); } + public void testSnapshotsUsedBySearchableSnapshotCannotBeDeleted() throws Exception { + final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final Settings.Builder repositorySettings = randomRepositorySettings(); + createRepository(repositoryName, FsRepository.TYPE, repositorySettings); + + final int nbIndices = randomIntBetween(1, 5); + final Map docsPerIndex = new HashMap<>(nbIndices); + final int nbSnapshots = randomIntBetween(1, 5); + final Map> indicesPerSnapshot = new HashMap<>(nbSnapshots); + final Set mountedSnapshots = new HashSet<>(); + + for (int i = 0; i < nbIndices; i++) { + final String indexName = "index-" + i; + createAndPopulateIndex( + indexName, + Settings.builder().put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1).put(INDEX_SOFT_DELETES_SETTING.getKey(), true) + ); + final TotalHits totalHits = internalCluster().client() + .prepareSearch(indexName) + .setTrackTotalHits(true) + .get() + .getHits() + .getTotalHits(); + docsPerIndex.put(indexName, totalHits.value); + } + + for (int i = 0; i < nbSnapshots; i++) { + final SnapshotInfo snapshotInfo = createSnapshot( + repositoryName, + "snapshot-" + i, + randomSubsetOf(between(1, nbIndices), docsPerIndex.keySet()) + ); + indicesPerSnapshot.put(snapshotInfo.snapshotId(), snapshotInfo.indices()); + + if (randomBoolean()) { + final String snapshotName = snapshotInfo.snapshotId().getName(); + final String cloneSnapshotName = "clone-" + snapshotName; + assertAcked( + client().admin() + .cluster() + .prepareCloneSnapshot(repositoryName, snapshotName, cloneSnapshotName) + .setIndices( + randomSubsetOf(between(1, snapshotInfo.indices().size()), snapshotInfo.indices()).toArray(String[]::new) + ) + .get() + ); + + final GetSnapshotsResponse snapshots = client().admin() + .cluster() + .prepareGetSnapshots(repositoryName) + .addSnapshots(cloneSnapshotName) + .get(); + final SnapshotInfo cloneSnapshotInfo = snapshots.getSnapshots().get(0); + assertThat(cloneSnapshotInfo, notNullValue()); + assertThat(cloneSnapshotInfo.snapshotId().getName(), equalTo(cloneSnapshotName)); + indicesPerSnapshot.put(cloneSnapshotInfo.snapshotId(), cloneSnapshotInfo.indices()); + } + } + + assertAcked(client().admin().indices().prepareDelete("index-*")); + + for (Map.Entry> snapshot : indicesPerSnapshot.entrySet()) { + final String snapshotName = snapshot.getKey().getName(); + if (indicesPerSnapshot.size() == 1 || randomBoolean()) { + for (String index : snapshot.getValue()) { + Storage storage = randomFrom(Storage.values()); + String restoredIndexName = "mounted-" + snapshotName + '-' + index + '-' + storage.name().toLowerCase(Locale.ROOT); + mountSnapshot(repositoryName, snapshotName, index, restoredIndexName, Settings.EMPTY, storage); + assertHitCount(client().prepareSearch(restoredIndexName).setTrackTotalHits(true).get(), docsPerIndex.get(index)); + } + mountedSnapshots.add(snapshot.getKey()); + } + } + + for (Map.Entry> snapshot : indicesPerSnapshot.entrySet()) { + final SnapshotId snapshotId = snapshot.getKey(); + if (mountedSnapshots.contains(snapshotId) == false) { + assertAcked(clusterAdmin().prepareDeleteSnapshot(repositoryName, snapshotId.getName()).get()); + } else { + SnapshotException exception = expectThrows( + SnapshotException.class, + () -> clusterAdmin().prepareDeleteSnapshot(repositoryName, snapshotId.getName()).get() + ); + assertThat( + exception.getMessage(), + containsString( + "[" + + repositoryName + + ':' + + snapshotId + + "] found " + + snapshot.getValue().size() + + " searchable snapshots indices that use the snapshot:" + ) + ); + } + } + + assertAcked(client().admin().indices().prepareDelete("mounted-*")); + + for (SnapshotId snapshotId : indicesPerSnapshot.keySet()) { + if (mountedSnapshots.contains(snapshotId)) { + assertAcked(clusterAdmin().prepareDeleteSnapshot(repositoryName, snapshotId.getName()).get()); + } + } + + final GetSnapshotsResponse snapshots = clusterAdmin().prepareGetSnapshots(repositoryName).get(); + assertThat(snapshots.getSnapshots(), hasSize(0)); + } + private static Settings deleteSnapshotIndexSettings(boolean value) { return Settings.builder().put(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, value).build(); }