Skip to content

Commit cf61996

Browse files
dakronedakrone
authored
Auto convert shared_cache SBIs to only use frozen tier (#71014)
This commit converts the index metadata of searchable snapshot indices using the `shared_cache` storage type to: - Remove all the `index.routing.allocation.(include|exclude|require)._tier` settings - Sets `index.routing.allocation.include._tier_preference` to `data_frozen` automatically when the index metadata is read This is in preperation to enforcing that the `_tier_preference` setting is always set to `data_frozen` for shared cache SBIs. Relates to #70846, #71013, #70786, #70141
1 parent 9ab1a6c commit cf61996

File tree

3 files changed

+128
-9
lines changed

3 files changed

+128
-9
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,11 @@ public IndexMetadataVerifier(Settings settings, NamedXContentRegistry xContentRe
7171
public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, Version minimumIndexCompatibilityVersion) {
7272
checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion);
7373

74-
// we have to run this first otherwise in we try to create IndexSettings
75-
// with broken settings and fail in checkMappingsCompatibility
76-
IndexMetadata newMetadata = archiveBrokenIndexSettings(indexMetadata);
74+
// First convert any shared_cache searchable snapshot indices to only use _tier_preference: data_frozen
75+
IndexMetadata newMetadata = convertSharedCacheTierPreference(indexMetadata);
76+
// Next we have to run this otherwise if we try to create IndexSettings
77+
// with broken settings it would fail in checkMappingsCompatibility
78+
newMetadata = archiveBrokenIndexSettings(newMetadata);
7779
checkMappingsCompatibility(newMetadata);
7880
return newMetadata;
7981
}
@@ -187,4 +189,30 @@ IndexMetadata archiveBrokenIndexSettings(IndexMetadata indexMetadata) {
187189
return indexMetadata;
188190
}
189191
}
192+
193+
/**
194+
* Convert shared_cache searchable snapshot indices to only specify
195+
* _tier_preference: data_frozen, removing any pre-existing tier allocation rules.
196+
*/
197+
IndexMetadata convertSharedCacheTierPreference(IndexMetadata indexMetadata) {
198+
final Settings settings = indexMetadata.getSettings();
199+
// Only remove these settings for a shared_cache searchable snapshot
200+
if ("snapshot".equals(settings.get("index.store.type", "")) && settings.getAsBoolean("index.store.snapshot.partial", false)) {
201+
final Settings.Builder settingsBuilder = Settings.builder().put(settings);
202+
// Clear any allocation rules other than preference for tier
203+
settingsBuilder.remove("index.routing.allocation.include._tier");
204+
settingsBuilder.remove("index.routing.allocation.exclude._tier");
205+
settingsBuilder.remove("index.routing.allocation.require._tier");
206+
// Override the tier preference to be only on frozen nodes, regardless of its current setting
207+
settingsBuilder.put("index.routing.allocation.include._tier_preference", "data_frozen");
208+
final Settings newSettings = settingsBuilder.build();
209+
if (settings.equals(newSettings)) {
210+
return indexMetadata;
211+
} else {
212+
return IndexMetadata.builder(indexMetadata).settings(newSettings).build();
213+
}
214+
} else {
215+
return indexMetadata;
216+
}
217+
}
190218
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataConversionTests.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.cluster.metadata;
9+
10+
import org.elasticsearch.common.UUIDs;
11+
import org.elasticsearch.common.settings.IndexScopedSettings;
12+
import org.elasticsearch.common.settings.Settings;
13+
import org.elasticsearch.index.IndexModule;
14+
import org.elasticsearch.index.mapper.MapperRegistry;
15+
import org.elasticsearch.plugins.MapperPlugin;
16+
import org.elasticsearch.test.ESTestCase;
17+
import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants;
18+
19+
import java.util.Collections;
20+
21+
import static org.elasticsearch.test.VersionUtils.randomIndexCompatibleVersion;
22+
import static org.hamcrest.Matchers.equalTo;
23+
24+
public class IndexMetadataConversionTests extends ESTestCase {
25+
26+
public void testConvertSearchableSnapshotSettings() {
27+
IndexMetadataVerifier service = getIndexMetadataVerifier();
28+
IndexMetadata src = newIndexMeta("foo", Settings.EMPTY);
29+
IndexMetadata indexMetadata = service.convertSharedCacheTierPreference(src);
30+
assertSame(indexMetadata, src);
31+
32+
// A full_copy searchable snapshot (settings should be untouched)
33+
src = newIndexMeta("foo", Settings.builder()
34+
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
35+
.put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), false)
36+
.put("index.routing.allocation.include._tier", "data_hot")
37+
.put("index.routing.allocation.exclude._tier", "data_warm")
38+
.put("index.routing.allocation.require._tier", "data_hot")
39+
.put("index.routing.allocation.include._tier_preference", "data_cold")
40+
.build());
41+
indexMetadata = service.convertSharedCacheTierPreference(src);
42+
assertSame(indexMetadata, src);
43+
44+
// A shared_cache searchable snapshot with valid settings (metadata should be untouched)
45+
src = newIndexMeta("foo", Settings.builder()
46+
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
47+
.put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), false)
48+
.put("index.routing.allocation.include._tier_preference", "data_frozen")
49+
.build());
50+
indexMetadata = service.convertSharedCacheTierPreference(src);
51+
assertSame(indexMetadata, src);
52+
53+
// A shared_cache searchable snapshot (should have its settings converted)
54+
src = newIndexMeta("foo", Settings.builder()
55+
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
56+
.put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), true)
57+
.put("index.routing.allocation.include._tier", "data_hot")
58+
.put("index.routing.allocation.exclude._tier", "data_warm")
59+
.put("index.routing.allocation.require._tier", "data_hot")
60+
.put("index.routing.allocation.include._tier_preference", "data_frozen,data_cold")
61+
.build());
62+
indexMetadata = service.convertSharedCacheTierPreference(src);
63+
assertNotSame(indexMetadata, src);
64+
Settings newSettings = indexMetadata.getSettings();
65+
assertNull(newSettings.get("index.routing.allocation.include._tier"));
66+
assertNull(newSettings.get("index.routing.allocation.exclude._tier"));
67+
assertNull(newSettings.get("index.routing.allocation.require._tier"));
68+
assertThat(newSettings.get("index.routing.allocation.include._tier_preference"), equalTo("data_frozen"));
69+
}
70+
71+
private IndexMetadataVerifier getIndexMetadataVerifier() {
72+
return new IndexMetadataVerifier(
73+
Settings.EMPTY,
74+
xContentRegistry(),
75+
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(),
76+
MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
77+
null
78+
);
79+
}
80+
81+
public static IndexMetadata newIndexMeta(String name, Settings indexSettings) {
82+
final Settings settings = Settings.builder()
83+
.put(IndexMetadata.SETTING_VERSION_CREATED, randomIndexCompatibleVersion(random()))
84+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 5))
85+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5))
86+
.put(IndexMetadata.SETTING_CREATION_DATE, randomNonNegativeLong())
87+
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
88+
.put(indexSettings)
89+
.build();
90+
final IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(name).settings(settings);
91+
if (randomBoolean()) {
92+
indexMetadataBuilder.state(IndexMetadata.State.CLOSE);
93+
}
94+
return indexMetadataBuilder.build();
95+
}
96+
}

‎x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -500,12 +500,7 @@ public void testCreateAndRestorePartialSearchableSnapshot() throws Exception {
500500
expectedReplicas = 0;
501501
}
502502
final String expectedDataTiersPreference;
503-
if (randomBoolean()) {
504-
expectedDataTiersPreference = String.join(",", randomSubsetOf(DataTier.ALL_DATA_TIERS));
505-
indexSettingsBuilder.put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, expectedDataTiersPreference);
506-
} else {
507-
expectedDataTiersPreference = getDataTiersPreference(MountSearchableSnapshotRequest.Storage.SHARED_CACHE);
508-
}
503+
expectedDataTiersPreference = getDataTiersPreference(MountSearchableSnapshotRequest.Storage.SHARED_CACHE);
509504

510505
indexSettingsBuilder.put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.ZERO);
511506
final AtomicBoolean statsWatcherRunning = new AtomicBoolean(true);

0 commit comments

Comments
 (0)