Skip to content

Commit 6621511

Browse files
tlrxtlrx
committed
Introduce searchable snapshots index setting for cascade deletion of snapshots
1 parent 744c75d commit 6621511

File tree

4 files changed

+289
-5
lines changed

4 files changed

+289
-5
lines changed

‎server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C
8383

8484
public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY = "index.store.snapshot.repository_name";
8585
public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY = "index.store.snapshot.repository_uuid";
86+
public static final String SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY = "index.store.snapshot.snapshot_uuid";
87+
public static final String SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION = "index.store.snapshot.delete_searchable_snapshot";
8688

8789
private final Map<String, Repository.Factory> typesRegistry;
8890
private final Map<String, Repository.Factory> internalTypesRegistry;

‎server/src/main/java/org/elasticsearch/snapshots/RestoreService.java‎

Lines changed: 115 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.elasticsearch.cluster.routing.allocation.AllocationService;
5252
import org.elasticsearch.cluster.service.ClusterService;
5353
import org.elasticsearch.common.Priority;
54+
import org.elasticsearch.common.Strings;
5455
import org.elasticsearch.common.UUIDs;
5556
import org.elasticsearch.common.collect.ImmutableOpenMap;
5657
import org.elasticsearch.common.logging.DeprecationCategory;
@@ -83,6 +84,7 @@
8384
import java.util.HashMap;
8485
import java.util.HashSet;
8586
import java.util.List;
87+
import java.util.Locale;
8688
import java.util.Map;
8789
import java.util.Objects;
8890
import java.util.Optional;
@@ -101,6 +103,9 @@
101103
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
102104
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
103105
import static org.elasticsearch.common.util.set.Sets.newHashSet;
106+
import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
107+
import static org.elasticsearch.repositories.RepositoriesService.SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION;
108+
import static org.elasticsearch.repositories.RepositoriesService.SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY;
104109
import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices;
105110
import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE;
106111

@@ -1141,6 +1146,9 @@ public ClusterState execute(ClusterState currentState) {
11411146
resolveSystemIndicesToDelete(currentState, featureStatesToRestore)
11421147
);
11431148

1149+
// List of searchable snapshots indices to restore
1150+
final Set<Index> searchableSnapshotsIndices = new HashSet<>();
1151+
11441152
// Updating cluster state
11451153
final Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata());
11461154
final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
@@ -1230,6 +1238,10 @@ && isSystemIndex(snapshotIndexMetadata) == false) {
12301238
: new ShardRestoreStatus(localNodeId)
12311239
);
12321240
}
1241+
1242+
if ("snapshot".equals(INDEX_STORE_TYPE_SETTING.get(updatedIndexMetadata.getSettings()))) {
1243+
searchableSnapshotsIndices.add(updatedIndexMetadata.getIndex());
1244+
}
12331245
}
12341246

12351247
final ClusterState.Builder builder = ClusterState.builder(currentState);
@@ -1267,10 +1279,11 @@ && isSystemIndex(snapshotIndexMetadata) == false) {
12671279
}
12681280

12691281
updater.accept(currentState, mdBuilder);
1270-
return allocationService.reroute(
1271-
builder.metadata(mdBuilder).blocks(blocks).routingTable(rtBuilder.build()).build(),
1272-
"restored snapshot [" + snapshot + "]"
1273-
);
1282+
final ClusterState updatedClusterState = builder.metadata(mdBuilder).blocks(blocks).routingTable(rtBuilder.build()).build();
1283+
if (searchableSnapshotsIndices.isEmpty() == false) {
1284+
ensureSearchableSnapshotsRestorable(updatedClusterState, snapshotInfo, searchableSnapshotsIndices);
1285+
}
1286+
return allocationService.reroute(updatedClusterState, "restored snapshot [" + snapshot + "]");
12741287
}
12751288

12761289
private void applyDataStreamRestores(ClusterState currentState, Metadata.Builder mdBuilder) {
@@ -1488,4 +1501,102 @@ private void ensureValidIndexName(ClusterState currentState, IndexMetadata snaps
14881501
createIndexService.validateDotIndex(renamedIndexName, isHidden);
14891502
createIndexService.validateIndexSettings(renamedIndexName, snapshotIndexMetadata.getSettings(), false);
14901503
}
1504+
1505+
private static void ensureSearchableSnapshotsRestorable(
1506+
final ClusterState currentState,
1507+
final SnapshotInfo snapshotInfo,
1508+
final Set<Index> indices
1509+
) {
1510+
final Metadata metadata = currentState.metadata();
1511+
for (Index index : indices) {
1512+
final Settings indexSettings = metadata.getIndexSafe(index).getSettings();
1513+
assert "snapshot".equals(INDEX_STORE_TYPE_SETTING.get(indexSettings)) : "not a snapshot backed index: " + index;
1514+
1515+
final String repositoryUuid = indexSettings.get(RepositoriesService.SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY);
1516+
final String repositoryName = indexSettings.get(RepositoriesService.SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY);
1517+
final String snapshotUuid = indexSettings.get(SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY);
1518+
1519+
final boolean deleteSnapshot = indexSettings.getAsBoolean(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, false);
1520+
if (deleteSnapshot && snapshotInfo.indices().size() != 1) {
1521+
throw new SnapshotRestoreException(
1522+
repositoryName,
1523+
snapshotInfo.snapshotId().getName(),
1524+
String.format(
1525+
Locale.ROOT,
1526+
"cannot mount snapshot [%s/%s:%s] as index [%s] with the deletion of snapshot on index removal enabled "
1527+
+ "[index.store.snapshot.delete_searchable_snapshot: true]; snapshot contains [%d] indices instead of 1.",
1528+
repositoryName,
1529+
repositoryUuid,
1530+
snapshotInfo.snapshotId().getName(),
1531+
index.getName(),
1532+
snapshotInfo.indices().size()
1533+
)
1534+
);
1535+
}
1536+
1537+
for (IndexMetadata other : metadata) {
1538+
if (other.getIndex().equals(index)) {
1539+
continue; // do not check the searchable snapshot index against itself
1540+
}
1541+
final Settings otherSettings = other.getSettings();
1542+
if ("snapshot".equals(INDEX_STORE_TYPE_SETTING.get(otherSettings)) == false) {
1543+
continue; // other index is not a searchable snapshot index, skip
1544+
}
1545+
final String otherSnapshotUuid = otherSettings.get(SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY);
1546+
if (Objects.equals(snapshotUuid, otherSnapshotUuid) == false) {
1547+
continue; // other index is backed by a different snapshot, skip
1548+
}
1549+
final String otherRepositoryUuid = otherSettings.get(RepositoriesService.SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY);
1550+
final String otherRepositoryName = otherSettings.get(RepositoriesService.SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY);
1551+
if (matchRepository(repositoryUuid, repositoryName, otherRepositoryUuid, otherRepositoryName) == false) {
1552+
continue; // other index is backed by a snapshot from a different repository, skip
1553+
}
1554+
if (otherSettings.getAsBoolean(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, false)) {
1555+
throw new SnapshotRestoreException(
1556+
repositoryName,
1557+
snapshotInfo.snapshotId().getName(),
1558+
String.format(
1559+
Locale.ROOT,
1560+
"cannot mount snapshot [%s/%s:%s] as index [%s]; another index %s uses the snapshot "
1561+
+ "with the deletion of snapshot on index removal enabled "
1562+
+ "[index.store.snapshot.delete_searchable_snapshot: true].",
1563+
repositoryName,
1564+
repositoryUuid,
1565+
snapshotInfo.snapshotId().getName(),
1566+
index.getName(),
1567+
other.getIndex()
1568+
)
1569+
);
1570+
} else if (deleteSnapshot) {
1571+
throw new SnapshotRestoreException(
1572+
repositoryName,
1573+
snapshotInfo.snapshotId().getName(),
1574+
String.format(
1575+
Locale.ROOT,
1576+
"cannot mount snapshot [%s/%s:%s] as index [%s] with the deletion of snapshot on index removal enabled "
1577+
+ "[index.store.snapshot.delete_searchable_snapshot: true]; another index %s uses the snapshot.",
1578+
repositoryName,
1579+
repositoryUuid,
1580+
snapshotInfo.snapshotId().getName(),
1581+
index.getName(),
1582+
other.getIndex()
1583+
)
1584+
);
1585+
}
1586+
}
1587+
}
1588+
}
1589+
1590+
private static boolean matchRepository(
1591+
String repositoryUuid,
1592+
String repositoryName,
1593+
String otherRepositoryUuid,
1594+
String otherRepositoryName
1595+
) {
1596+
if (Strings.hasLength(repositoryUuid) && Strings.hasLength(otherRepositoryUuid)) {
1597+
return Objects.equals(repositoryUuid, otherRepositoryUuid);
1598+
} else {
1599+
return Objects.equals(repositoryName, otherRepositoryName);
1600+
}
1601+
}
14911602
}

‎x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,27 @@
99

1010
import org.apache.lucene.search.TotalHits;
1111
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
12+
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
13+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1214
import org.elasticsearch.common.settings.Settings;
1315
import org.elasticsearch.common.unit.ByteSizeValue;
1416
import org.elasticsearch.repositories.fs.FsRepository;
17+
import org.elasticsearch.snapshots.SnapshotRestoreException;
18+
import org.hamcrest.Matcher;
1519

1620
import java.util.Arrays;
1721
import java.util.List;
1822
import java.util.Locale;
1923

2024
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING;
2125
import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
26+
import static org.elasticsearch.repositories.RepositoriesService.SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION;
2227
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
2328
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
2429
import static org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest.Storage;
30+
import static org.hamcrest.Matchers.allOf;
2531
import static org.hamcrest.Matchers.containsString;
32+
import static org.hamcrest.Matchers.equalTo;
2633

2734
public class SearchableSnapshotsRepositoryIntegTests extends BaseFrozenSearchableSnapshotsIntegTestCase {
2835

@@ -110,4 +117,153 @@ public void testRepositoryUsedBySearchableSnapshotCanBeUpdatedButNotUnregistered
110117

111118
assertAcked(clusterAdmin().prepareDeleteRepository(updatedRepositoryName));
112119
}
120+
121+
public void testMountIndexWithDeletionOfSnapshotFailsIfNotSingleIndexSnapshot() throws Exception {
122+
final String repository = "repository-" + getTestName().toLowerCase(Locale.ROOT);
123+
createRepository(repository, FsRepository.TYPE, randomRepositorySettings());
124+
125+
final int nbIndices = randomIntBetween(2, 5);
126+
for (int i = 0; i < nbIndices; i++) {
127+
createAndPopulateIndex(
128+
"index-" + i,
129+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(INDEX_SOFT_DELETES_SETTING.getKey(), true)
130+
);
131+
}
132+
133+
final String snapshot = "snapshot";
134+
createFullSnapshot(repository, snapshot);
135+
assertAcked(client().admin().indices().prepareDelete("index-*"));
136+
137+
final String index = "index-" + randomInt(nbIndices - 1);
138+
final String mountedIndex = "mounted-" + index;
139+
140+
final SnapshotRestoreException exception = expectThrows(
141+
SnapshotRestoreException.class,
142+
() -> mountSnapshot(repository, snapshot, index, mountedIndex, deleteSnapshotIndexSettings(true), randomFrom(Storage.values()))
143+
);
144+
assertThat(
145+
exception.getMessage(),
146+
allOf(
147+
containsString("cannot mount snapshot [" + repository + '/'),
148+
containsString(snapshot + "] as index [" + mountedIndex + "] with the deletion of snapshot on index removal enabled"),
149+
containsString("[index.store.snapshot.delete_searchable_snapshot: true]; "),
150+
containsString("snapshot contains [" + nbIndices + "] indices instead of 1.")
151+
)
152+
);
153+
}
154+
155+
public void testMountIndexWithDeletionOfSnapshot() throws Exception {
156+
final String repository = "repository-" + getTestName().toLowerCase(Locale.ROOT);
157+
createRepository(repository, FsRepository.TYPE, randomRepositorySettings());
158+
159+
final String index = "index";
160+
createAndPopulateIndex(index, Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true));
161+
162+
final TotalHits totalHits = internalCluster().client().prepareSearch(index).setTrackTotalHits(true).get().getHits().getTotalHits();
163+
164+
final String snapshot = "snapshot";
165+
createSnapshot(repository, snapshot, List.of(index));
166+
assertAcked(client().admin().indices().prepareDelete(index));
167+
168+
String mounted = "mounted-with-setting-enabled";
169+
mountSnapshot(repository, snapshot, index, mounted, deleteSnapshotIndexSettings(true), randomFrom(Storage.values()));
170+
assertIndexSetting(mounted, SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, equalTo("true"));
171+
assertHitCount(client().prepareSearch(mounted).setTrackTotalHits(true).get(), totalHits.value);
172+
173+
// the snapshot is already mounted as an index with "index.store.snapshot.delete_searchable_snapshot: true",
174+
// any attempt to mount the snapshot again should fail
175+
final String mountedAgain = randomValueOtherThan(mounted, () -> randomAlphaOfLength(10).toLowerCase(Locale.ROOT));
176+
SnapshotRestoreException exception = expectThrows(
177+
SnapshotRestoreException.class,
178+
() -> mountSnapshot(repository, snapshot, index, mountedAgain, deleteSnapshotIndexSettings(randomBoolean()))
179+
);
180+
assertThat(
181+
exception.getMessage(),
182+
allOf(
183+
containsString("cannot mount snapshot [" + repository + '/'),
184+
containsString(':' + snapshot + "] as index [" + mountedAgain + "]; another index [" + mounted + '/'),
185+
containsString("] uses the snapshot with the deletion of snapshot on index removal enabled "),
186+
containsString("[index.store.snapshot.delete_searchable_snapshot: true].")
187+
)
188+
);
189+
190+
assertAcked(client().admin().indices().prepareDelete(mounted));
191+
mounted = "mounted-with-setting-disabled";
192+
mountSnapshot(repository, snapshot, index, mounted, deleteSnapshotIndexSettings(false), randomFrom(Storage.values()));
193+
assertIndexSetting(mounted, SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, equalTo("false"));
194+
assertHitCount(client().prepareSearch(mounted).setTrackTotalHits(true).get(), totalHits.value);
195+
196+
// the snapshot is now mounted as an index with "index.store.snapshot.delete_searchable_snapshot: false",
197+
// any attempt to mount the snapshot again with "delete_searchable_snapshot: true" should fail
198+
exception = expectThrows(
199+
SnapshotRestoreException.class,
200+
() -> mountSnapshot(repository, snapshot, index, mountedAgain, deleteSnapshotIndexSettings(true))
201+
);
202+
assertThat(
203+
exception.getMessage(),
204+
allOf(
205+
containsString("cannot mount snapshot [" + repository + '/'),
206+
containsString(snapshot + "] as index [" + mountedAgain + "] with the deletion of snapshot on index removal enabled"),
207+
containsString("[index.store.snapshot.delete_searchable_snapshot: true]; another index [" + mounted + '/'),
208+
containsString("] uses the snapshot.")
209+
)
210+
);
211+
212+
// but we can continue to mount the snapshot, as long as it does not require the cascade deletion of the snapshot
213+
mountSnapshot(repository, snapshot, index, mountedAgain, deleteSnapshotIndexSettings(false));
214+
assertIndexSetting(mountedAgain, SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, equalTo("false"));
215+
assertHitCount(client().prepareSearch(mountedAgain).setTrackTotalHits(true).get(), totalHits.value);
216+
217+
assertAcked(client().admin().indices().prepareDelete(mountedAgain));
218+
assertAcked(client().admin().indices().prepareDelete(mounted));
219+
}
220+
221+
public void testDeletionOfSnapshotSettingCannotBeUpdated() throws Exception {
222+
final String repository = "repository-" + getTestName().toLowerCase(Locale.ROOT);
223+
createRepository(repository, FsRepository.TYPE, randomRepositorySettings());
224+
225+
final String index = "index";
226+
createAndPopulateIndex(index, Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true));
227+
228+
final TotalHits totalHits = internalCluster().client().prepareSearch(index).setTrackTotalHits(true).get().getHits().getTotalHits();
229+
230+
final String snapshot = "snapshot";
231+
createSnapshot(repository, snapshot, List.of(index));
232+
assertAcked(client().admin().indices().prepareDelete(index));
233+
234+
final String mounted = "mounted-" + index;
235+
final boolean deleteSnapshot = randomBoolean();
236+
237+
mountSnapshot(repository, snapshot, index, mounted, deleteSnapshotIndexSettings(deleteSnapshot), randomFrom(Storage.values()));
238+
assertIndexSetting(mounted, SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, equalTo(Boolean.toString(deleteSnapshot)));
239+
assertHitCount(client().prepareSearch(mounted).setTrackTotalHits(true).get(), totalHits.value);
240+
241+
final IllegalArgumentException exception = expectThrows(
242+
IllegalArgumentException.class,
243+
() -> client().admin()
244+
.indices()
245+
.prepareUpdateSettings(mounted)
246+
.setSettings(deleteSnapshotIndexSettings(deleteSnapshot == false))
247+
.get()
248+
);
249+
assertThat(
250+
exception.getMessage(),
251+
containsString("can not update private setting [index.store.snapshot.delete_searchable_snapshot]; ")
252+
);
253+
254+
assertAcked(client().admin().indices().prepareDelete(mounted));
255+
}
256+
257+
private static Settings deleteSnapshotIndexSettings(boolean value) {
258+
return Settings.builder().put(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, value).build();
259+
}
260+
261+
private static void assertIndexSetting(String indexName, String indexSettingName, Matcher<String> matcher) {
262+
final GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(indexName).get();
263+
assertThat(
264+
"Unexpected value for setting [" + indexSettingName + "] of index [" + indexName + ']',
265+
getSettingsResponse.getSetting(indexName, indexSettingName),
266+
matcher
267+
);
268+
}
113269
}

0 commit comments

Comments
 (0)