Prevent deletion of snapshots that are used by snapshot backed indices

server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java

@@ -771,7 +771,7 @@ private static void ensureNoSearchableSnapshotsIndicesInUse(ClusterState cluster
         }
     }
 
-    private static boolean indexSettingsMatchRepositoryMetadata(IndexMetadata indexMetadata, RepositoryMetadata repositoryMetadata) {
+    public static boolean indexSettingsMatchRepositoryMetadata(IndexMetadata indexMetadata, RepositoryMetadata repositoryMetadata) {
         if (indexMetadata.isSearchableSnapshot()) {
             final Settings indexSettings = indexMetadata.getSettings();
             final String indexRepositoryUuid = indexSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY);

server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java

@@ -47,6 +47,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
+import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -114,6 +115,7 @@
 
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.cluster.SnapshotsInProgress.completed;
+import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY;
 
 /**
  * Service responsible for creating snapshots. This service runs all the steps executed on the master node during snapshot creation and
@@ -2158,6 +2160,11 @@ public ClusterState execute(ClusterState currentState) {
                         );
                     }
                 }
+
+                for (SnapshotId snapshotId : snapshotIds) {
+                    ensureNoSearchableSnapshotsIndicesInUse(repository.getMetadata(), snapshotId, currentState);
+                }
+
                 // Snapshot ids that will have to be physically deleted from the repository
                 final Set<SnapshotId> snapshotIdsRequiringCleanup = new HashSet<>(snapshotIds);
                 final SnapshotsInProgress updatedSnapshots = snapshotsInProgress.withUpdatedEntriesForRepo(
@@ -2956,6 +2963,35 @@ static Map<String, DataStreamAlias> filterDataStreamAliases(
             .collect(Collectors.toMap(DataStreamAlias::getName, Function.identity()));
     }
 
+    private static void ensureNoSearchableSnapshotsIndicesInUse(RepositoryMetadata repository, SnapshotId snapshot, ClusterState state) {
+        long count = 0L;
+        List<Index> indices = null;
+        for (IndexMetadata indexMetadata : state.metadata()) {
+            final Settings indexSettings = indexMetadata.getSettings();
+            if (RepositoriesService.indexSettingsMatchRepositoryMetadata(indexMetadata, repository)
+                && Objects.equals(snapshot.getUUID(), indexSettings.get(SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY))) {
+                if (indices == null) {
+                    indices = new ArrayList<>();
+                }
+                if (indices.size() < 5) {
+                    indices.add(indexMetadata.getIndex());
+                }
+                count += 1L;
+            }
+        }
+        if (indices != null && indices.isEmpty() == false) {
+            throw new SnapshotException(
+                repository.name(),
+                snapshot.toString(),
+                "found "
+                    + count
+                    + " searchable snapshots indices that use the snapshot: "
+                    + Strings.collectionToCommaDelimitedString(indices)
+                    + (count > indices.size() ? ",..." : "")
+            );
+        }
+    }
+
     /**
      * Adds snapshot completion listener
      *

x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/repository.yml

@@ -54,7 +54,7 @@ setup:
 ---
 "Delete repository when a snapshot is mounted as an index":
   - skip:
-      version: " - 7.99.99"
+      version: " - 7.13.99"
       reason: "starting 8.0, an error occurs when a repository used by snapshot backed indices is deleted"
 
   - do:
@@ -93,3 +93,48 @@ setup:
       snapshot.delete_repository:
         repository: repository-fs
 
+
+---
+"Delete snapshot when the snapshot is mounted as an index":
+  - skip:
+      version: " - 7.13.99"
+      reason: "starting 8.0, an error occurs when a snapshot used by snapshot backed indices is deleted"
+
+  - do:
+      searchable_snapshots.mount:
+        repository: repository-fs
+        snapshot: snapshot
+        wait_for_completion: true
+        body:
+          index: docs
+          renamed_index: mounted-docs
+
+  - match: { snapshot.snapshot: snapshot }
+  - match: { snapshot.shards.failed: 0 }
+  - match: { snapshot.shards.successful: 1 }
+
+  # Returns an illegal state exception
+  - do:
+      catch: request
+      snapshot.delete:
+        repository: repository-fs
+        snapshot: snapshot
+
+  - do:
+      search:
+        index: mounted-docs
+        body:
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 3 }
+
+  - do:
+      indices.delete:
+        index: mounted-docs
+
+  - do:
+      snapshot.delete:
+        repository: repository-fs
+        snapshot: snapshot
+

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.searchablesnapshots;
 
 import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -16,12 +17,17 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.repositories.RepositoryConflictException;
 import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.snapshots.SnapshotException;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotRestoreException;
 
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING;
@@ -34,6 +40,8 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class SearchableSnapshotsRepositoryIntegTests extends BaseFrozenSearchableSnapshotsIntegTestCase {
@@ -60,8 +68,7 @@ public void testRepositoryUsedBySearchableSnapshotCanBeUpdatedButNotUnregistered
         createSnapshot(repositoryName, snapshotName, List.of(indexName));
         assertAcked(client().admin().indices().prepareDelete(indexName));
 
-        final int nbMountedIndices = 1;
-        randomIntBetween(1, 5);
+        final int nbMountedIndices = randomIntBetween(1, 5);
         final String[] mountedIndices = new String[nbMountedIndices];
 
         for (int i = 0; i < nbMountedIndices; i++) {
@@ -402,6 +409,116 @@ public void testRestoreSearchableSnapshotIndexWithDifferentSettingsConflicts() t
         assertAcked(client().admin().indices().prepareDelete("restored-with-same-setting-*"));
     }
 
+    public void testSnapshotsUsedBySearchableSnapshotCannotBeDeleted() throws Exception {
+        final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        final Settings.Builder repositorySettings = randomRepositorySettings();
+        createRepository(repositoryName, FsRepository.TYPE, repositorySettings);
+
+        final int nbIndices = randomIntBetween(1, 5);
+        final Map<String, Long> docsPerIndex = new HashMap<>(nbIndices);
+        final int nbSnapshots = randomIntBetween(1, 5);
+        final Map<SnapshotId, List<String>> indicesPerSnapshot = new HashMap<>(nbSnapshots);
+        final Set<SnapshotId> mountedSnapshots = new HashSet<>();
+
+        for (int i = 0; i < nbIndices; i++) {
+            final String indexName = "index-" + i;
+            createAndPopulateIndex(
+                indexName,
+                Settings.builder().put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1).put(INDEX_SOFT_DELETES_SETTING.getKey(), true)
+            );
+            final TotalHits totalHits = internalCluster().client()
+                .prepareSearch(indexName)
+                .setTrackTotalHits(true)
+                .get()
+                .getHits()
+                .getTotalHits();
+            docsPerIndex.put(indexName, totalHits.value);
+        }
+
+        for (int i = 0; i < nbSnapshots; i++) {
+            final SnapshotInfo snapshotInfo = createSnapshot(
+                repositoryName,
+                "snapshot-" + i,
+                randomSubsetOf(between(1, nbIndices), docsPerIndex.keySet())
+            );
+            indicesPerSnapshot.put(snapshotInfo.snapshotId(), snapshotInfo.indices());
+
+            if (randomBoolean()) {
+                final String snapshotName = snapshotInfo.snapshotId().getName();
+                final String cloneSnapshotName = "clone-" + snapshotName;
+                assertAcked(
+                    client().admin()
+                        .cluster()
+                        .prepareCloneSnapshot(repositoryName, snapshotName, cloneSnapshotName)
+                        .setIndices(
+                            randomSubsetOf(between(1, snapshotInfo.indices().size()), snapshotInfo.indices()).toArray(String[]::new)
+                        )
+                        .get()
+                );
+
+                final GetSnapshotsResponse snapshots = client().admin()
+                    .cluster()
+                    .prepareGetSnapshots(repositoryName)
+                    .addSnapshots(cloneSnapshotName)
+                    .get();
+                final SnapshotInfo cloneSnapshotInfo = snapshots.getSnapshots().get(0);
+                assertThat(cloneSnapshotInfo, notNullValue());
+                assertThat(cloneSnapshotInfo.snapshotId().getName(), equalTo(cloneSnapshotName));
+                indicesPerSnapshot.put(cloneSnapshotInfo.snapshotId(), cloneSnapshotInfo.indices());
+            }
+        }
+
+        assertAcked(client().admin().indices().prepareDelete("index-*"));
+
+        for (Map.Entry<SnapshotId, List<String>> snapshot : indicesPerSnapshot.entrySet()) {
+            final String snapshotName = snapshot.getKey().getName();
+            if (indicesPerSnapshot.size() == 1 || randomBoolean()) {
+                for (String index : snapshot.getValue()) {
+                    Storage storage = randomFrom(Storage.values());
+                    String restoredIndexName = "mounted-" + snapshotName + '-' + index + '-' + storage.name().toLowerCase(Locale.ROOT);
+                    mountSnapshot(repositoryName, snapshotName, index, restoredIndexName, Settings.EMPTY, storage);
+                    assertHitCount(client().prepareSearch(restoredIndexName).setTrackTotalHits(true).get(), docsPerIndex.get(index));
+                }
+                mountedSnapshots.add(snapshot.getKey());
+            }
+        }
+
+        for (Map.Entry<SnapshotId, List<String>> snapshot : indicesPerSnapshot.entrySet()) {
+            final SnapshotId snapshotId = snapshot.getKey();
+            if (mountedSnapshots.contains(snapshotId) == false) {
+                assertAcked(clusterAdmin().prepareDeleteSnapshot(repositoryName, snapshotId.getName()).get());
+            } else {
+                SnapshotException exception = expectThrows(
+                    SnapshotException.class,
+                    () -> clusterAdmin().prepareDeleteSnapshot(repositoryName, snapshotId.getName()).get()
+                );
+                assertThat(
+                    exception.getMessage(),
+                    containsString(
+                        "["
+                            + repositoryName
+                            + ':'
+                            + snapshotId
+                            + "] found "
+                            + snapshot.getValue().size()
+                            + " searchable snapshots indices that use the snapshot:"
+                    )
+                );
+            }
+        }
+
+        assertAcked(client().admin().indices().prepareDelete("mounted-*"));
+
+        for (SnapshotId snapshotId : indicesPerSnapshot.keySet()) {
+            if (mountedSnapshots.contains(snapshotId)) {
+                assertAcked(clusterAdmin().prepareDeleteSnapshot(repositoryName, snapshotId.getName()).get());
+            }
+        }
+
+        final GetSnapshotsResponse snapshots = clusterAdmin().prepareGetSnapshots(repositoryName).get();
+        assertThat(snapshots.getSnapshots(), hasSize(0));
+    }
+
     private static Settings deleteSnapshotIndexSettings(boolean value) {
         return Settings.builder().put(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, value).build();
     }