Skip to content

Commit 28fbaa7

Browse files
tlrxtlrx
authored
[7.x] Add maintenance service to clean up unused docs in snapshot blob cache (#78263)
* Add maintenance service to clean up unused docs in snapshot blob cache (#77686) Today we use the system index .snapshot-blob-cache to store parts of blobs and to avoid to fetch them again from the snapshot repository when recovering a searchable snapshot shard. This index is never cleaned up though and because it's a system index users won't be able to clean up manually in the future. This commit adds a BlobStoreCacheMaintenanceService which detects the deletion of searchable snapshot indices and triggers the deletion of associated documents in .snapshot-blob-cache. * fixes
1 parent dae177c commit 28fbaa7

File tree

6 files changed

+572
-25
lines changed

6 files changed

+572
-25
lines changed

‎x-pack/plugin/searchable-snapshots/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ dependencies {
1414
compileOnly project(path: xpackModule('core'))
1515
implementation project(path: 'preallocate')
1616
internalClusterTestImplementation(testArtifact(project(xpackModule('core'))))
17+
internalClusterTestImplementation(project(path: ':modules:reindex'))
1718
}
1819

1920
addQaCheckDependencies()

‎x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheIntegTests.java‎

Lines changed: 51 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
package org.elasticsearch.xpack.searchablesnapshots.cache.blob;
99

10+
import org.apache.lucene.store.AlreadyClosedException;
1011
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
1112
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
1213
import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -26,11 +27,14 @@
2627
import org.elasticsearch.common.settings.Settings;
2728
import org.elasticsearch.common.unit.ByteSizeUnit;
2829
import org.elasticsearch.common.unit.ByteSizeValue;
29-
import org.elasticsearch.common.util.CollectionUtils;
3030
import org.elasticsearch.common.xcontent.XContentBuilder;
3131
import org.elasticsearch.common.xcontent.XContentFactory;
3232
import org.elasticsearch.index.IndexNotFoundException;
33+
import org.elasticsearch.index.IndexService;
34+
import org.elasticsearch.index.reindex.ReindexPlugin;
35+
import org.elasticsearch.index.shard.IndexShard;
3336
import org.elasticsearch.index.shard.IndexingStats;
37+
import org.elasticsearch.indices.IndicesService;
3438
import org.elasticsearch.plugins.ClusterPlugin;
3539
import org.elasticsearch.plugins.Plugin;
3640
import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
@@ -62,6 +66,7 @@
6266
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
6367
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_BLOB_CACHE_INDEX;
6468
import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.SharedBytes.pageAligned;
69+
import static org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory.unwrapDirectory;
6570
import static org.hamcrest.Matchers.equalTo;
6671
import static org.hamcrest.Matchers.greaterThan;
6772

@@ -96,7 +101,10 @@ public static void tearDownCacheSettings() {
96101

97102
@Override
98103
protected Collection<Class<? extends Plugin>> nodePlugins() {
99-
return CollectionUtils.appendToCopy(super.nodePlugins(), WaitForSnapshotBlobCacheShardsActivePlugin.class);
104+
final List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
105+
plugins.add(WaitForSnapshotBlobCacheShardsActivePlugin.class);
106+
plugins.add(ReindexPlugin.class);
107+
return plugins;
100108
}
101109

102110
@Override
@@ -163,7 +171,7 @@ public void testBlobStoreCache() throws Exception {
163171
storage1,
164172
blobCacheMaxLength.getStringRep()
165173
);
166-
final String restoredIndex = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
174+
final String restoredIndex = "restored-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
167175
mountSnapshot(
168176
repositoryName,
169177
snapshot.getName(),
@@ -178,17 +186,9 @@ public void testBlobStoreCache() throws Exception {
178186
);
179187
ensureGreen(restoredIndex);
180188

181-
// wait for all async cache fills to complete
182-
assertBusy(() -> {
183-
for (final SearchableSnapshotShardStats shardStats : client().execute(
184-
SearchableSnapshotsStatsAction.INSTANCE,
185-
new SearchableSnapshotsStatsRequest()
186-
).actionGet().getStats()) {
187-
for (final SearchableSnapshotShardStats.CacheIndexInputStats indexInputStats : shardStats.getStats()) {
188-
assertThat(Strings.toString(indexInputStats), indexInputStats.getCurrentIndexCacheFills(), equalTo(0L));
189-
}
190-
}
191-
});
189+
assertRecoveryStats(restoredIndex, false);
190+
assertExecutorIsIdle(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME);
191+
waitForBlobCacheFillsToComplete();
192192

193193
for (final SearchableSnapshotShardStats shardStats : client().execute(
194194
SearchableSnapshotsStatsAction.INSTANCE,
@@ -213,11 +213,14 @@ public void testBlobStoreCache() throws Exception {
213213
equalTo("data_content,data_hot")
214214
);
215215

216+
refreshSystemIndex();
217+
216218
final long numberOfCachedBlobs = systemClient().prepareSearch(SNAPSHOT_BLOB_CACHE_INDEX)
217219
.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN)
218220
.get()
219221
.getHits()
220222
.getTotalHits().value;
223+
221224
IndexingStats indexingStats = systemClient().admin()
222225
.indices()
223226
.prepareStats(SNAPSHOT_BLOB_CACHE_INDEX)
@@ -231,19 +234,24 @@ public void testBlobStoreCache() throws Exception {
231234

232235
logger.info("--> verifying number of documents in index [{}]", restoredIndex);
233236
assertHitCount(client().prepareSearch(restoredIndex).setSize(0).setTrackTotalHits(true).get(), numberOfDocs);
234-
assertAcked(client().admin().indices().prepareDelete(restoredIndex));
235237

236-
assertBusy(() -> {
237-
refreshSystemIndex();
238-
assertThat(
239-
systemClient().prepareSearch(SNAPSHOT_BLOB_CACHE_INDEX).setSize(0).get().getHits().getTotalHits().value,
240-
greaterThan(0L)
241-
);
242-
});
238+
for (IndicesService indicesService : internalCluster().getDataNodeInstances(IndicesService.class)) {
239+
for (IndexService indexService : indicesService) {
240+
if (indexService.index().getName().equals(restoredIndex)) {
241+
for (IndexShard indexShard : indexService) {
242+
try {
243+
unwrapDirectory(indexShard.store().directory()).clearStats();
244+
} catch (AlreadyClosedException ignore) {
245+
// ok to ignore these
246+
}
247+
}
248+
}
249+
}
250+
}
243251

244252
final Storage storage2 = randomFrom(Storage.values());
245253
logger.info("--> mount snapshot [{}] as an index for the second time [storage={}]", snapshot, storage2);
246-
final String restoredAgainIndex = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
254+
final String restoredAgainIndex = "restored-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
247255
mountSnapshot(
248256
repositoryName,
249257
snapshot.getName(),
@@ -258,6 +266,10 @@ public void testBlobStoreCache() throws Exception {
258266
);
259267
ensureGreen(restoredAgainIndex);
260268

269+
assertRecoveryStats(restoredAgainIndex, false);
270+
assertExecutorIsIdle(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME);
271+
waitForBlobCacheFillsToComplete();
272+
261273
logger.info("--> verifying shards of [{}] were started without using the blob store more than necessary", restoredAgainIndex);
262274
checkNoBlobStoreAccess(useSoftDeletes);
263275

@@ -293,6 +305,10 @@ public Settings onNodeStopped(String nodeName) throws Exception {
293305
});
294306
ensureGreen(restoredAgainIndex);
295307

308+
assertRecoveryStats(restoredAgainIndex, false);
309+
assertExecutorIsIdle(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME);
310+
waitForBlobCacheFillsToComplete();
311+
296312
logger.info("--> shards of [{}] should start without downloading bytes from the blob store", restoredAgainIndex);
297313
checkNoBlobStoreAccess(useSoftDeletes);
298314

@@ -315,8 +331,18 @@ public Settings onNodeStopped(String nodeName) throws Exception {
315331
logger.info("--> verifying number of documents in index [{}]", restoredAgainIndex);
316332
assertHitCount(client().prepareSearch(restoredAgainIndex).setSize(0).setTrackTotalHits(true).get(), numberOfDocs);
317333

318-
// TODO also test when the index is frozen
319-
// TODO also test when prewarming is enabled
334+
logger.info("--> deleting indices, maintenance service should clean up snapshot blob cache index");
335+
assertAcked(client().admin().indices().prepareDelete("restored-*"));
336+
assertBusy(() -> {
337+
refreshSystemIndex();
338+
assertHitCount(
339+
systemClient().prepareSearch(SNAPSHOT_BLOB_CACHE_INDEX)
340+
.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN)
341+
.setSize(0)
342+
.get(),
343+
0L
344+
);
345+
});
320346
}
321347

322348
private void checkNoBlobStoreAccess(boolean useSoftDeletes) {

0 commit comments

Comments
 (0)