Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/reference/searchable-snapshots/index.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -172,9 +172,9 @@ xpack.searchable.snapshot.shared_cache.size: 4TB
----

IMPORTANT: Currently, you can configure
`xpack.searchable.snapshot.shared_cache.size` on any node. In a future release,
you will only be able to configure this setting on nodes with the
<<data-frozen-node,`data_frozen`>> role.
`xpack.searchable.snapshot.shared_cache.size` on any node. However, if the cache size is set on any
node that does not have the <<data-frozen-node,`data_frozen`>> role, it will be treated as though it
is set to `0b`.

You can set `xpack.searchable.snapshot.shared_cache.size` to any size between a
couple of gigabytes up to 90% of available disk space. We only recommend higher
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
);
}
builder.put(
FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(),
FrozenCacheService.SNAPSHOT_CACHE_SIZE,
rarely()
? randomBoolean()
? new ByteSizeValue(randomIntBetween(0, 10), ByteSizeUnit.KB)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
// Use an unbound cache so we can recover the searchable snapshot completely all the times
.put(CacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES))
// Have a shared cache of reasonable size available on each node because tests randomize over frozen and cold allocation
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofMb(randomLongBetween(1, 10)))
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, ByteSizeValue.ofMb(randomLongBetween(1, 10)))
.build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ public static void setUpCacheSettings() {
builder.put(CacheService.SNAPSHOT_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(), blobCacheMaxLength);

// Frozen (shared cache) cache should be large enough to not cause direct reads
builder.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofMb(128));
builder.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, ByteSizeValue.ofMb(128));
// Align ranges to match the blob cache max length
builder.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), blobCacheMaxLength);
builder.put(FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), blobCacheMaxLength);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING;
import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING;
import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.SNAPSHOT_CACHE_SIZE;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;

Expand All @@ -62,7 +62,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
// default to no cache: the tests create nodes with a cache configured as needed
.put(SNAPSHOT_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ZERO)
.put(SNAPSHOT_CACHE_SIZE, ByteSizeValue.ZERO)
.build();
}

Expand Down Expand Up @@ -121,7 +121,7 @@ public void testPartialSearchableSnapshotNotAllocatedToNodesWithoutCache() throw
.getDecisions()
.stream()
.anyMatch(
d -> d.getExplanation().contains(SNAPSHOT_CACHE_SIZE_SETTING.getKey())
d -> d.getExplanation().contains(SNAPSHOT_CACHE_SIZE)
&& d.getExplanation().contains("frozen searchable snapshot shards cannot be allocated to this node")
)
);
Expand All @@ -133,9 +133,7 @@ public void testPartialSearchableSnapshotAllocatedToNodesWithCache() throws Exce

final List<String> newNodeNames = internalCluster().startDataOnlyNodes(
between(1, 3),
Settings.builder()
.put(SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(randomLongBetween(1, ByteSizeValue.ofMb(10).getBytes())))
.build()
Settings.builder().put(SNAPSHOT_CACHE_SIZE, new ByteSizeValue(randomLongBetween(1, ByteSizeValue.ofMb(10).getBytes()))).build()
);

final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get();
Expand Down Expand Up @@ -201,9 +199,7 @@ public void onFailure(Exception e) {

final List<String> newNodes = internalCluster().startDataOnlyNodes(
2,
Settings.builder()
.put(SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(randomLongBetween(1, ByteSizeValue.ofMb(10).getBytes())))
.build()
Settings.builder().put(SNAPSHOT_CACHE_SIZE, new ByteSizeValue(randomLongBetween(1, ByteSizeValue.ofMb(10).getBytes()))).build()
);
final ActionFuture<RestoreSnapshotResponse> responseFuture = client().execute(MountSearchableSnapshotAction.INSTANCE, req);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,10 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;

import java.io.IOException;

import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING;

public class FrozenCacheInfoNodeAction extends ActionType<FrozenCacheInfoResponse> {

public static final String NAME = FrozenCacheInfoAction.NAME + "[n]";
Expand Down Expand Up @@ -53,7 +52,7 @@ public static class TransportAction extends HandledTransportAction<Request, Froz
@Inject
public TransportAction(Settings settings, TransportService transportService, ActionFilters actionFilters) {
super(NAME, transportService, actionFilters, Request::new);
response = new FrozenCacheInfoResponse(SNAPSHOT_CACHE_SIZE_SETTING.get(settings).getBytes() > 0);
response = new FrozenCacheInfoResponse(FrozenCacheService.getSnapshotCacheSize(settings).getBytes() > 0);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheInfoService;

import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING;
import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING;
import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.SNAPSHOT_CACHE_SIZE;

public class HasFrozenCacheAllocationDecider extends AllocationDecider {

Expand All @@ -28,7 +28,7 @@ public class HasFrozenCacheAllocationDecider extends AllocationDecider {
private static final Decision STILL_FETCHING = Decision.single(
Decision.Type.THROTTLE,
NAME,
"value of [" + SNAPSHOT_CACHE_SIZE_SETTING.getKey() + "] on this node is not known yet"
"value of [" + SNAPSHOT_CACHE_SIZE + "] on this node is not known yet"
);

private static final Decision HAS_FROZEN_CACHE = Decision.single(
Expand All @@ -40,9 +40,7 @@ public class HasFrozenCacheAllocationDecider extends AllocationDecider {
private static final Decision NO_FROZEN_CACHE = Decision.single(
Decision.Type.NO,
NAME,
"node setting ["
+ SNAPSHOT_CACHE_SIZE_SETTING.getKey()
+ "] is set to zero, so frozen searchable snapshot shards cannot be allocated to this node"
"node setting [" + SNAPSHOT_CACHE_SIZE + "] is set to zero, so frozen searchable snapshot shards cannot be allocated to this node"
);

private static final Decision UNKNOWN_FROZEN_CACHE = Decision.single(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,25 @@ private static Setting.Validator<ByteSizeValue> getPageSizeAlignedByteSizeValueV
};
}

/**
* Returns the snapshot cache size to be used for the given settings, however, all non-frozen
* nodes (nodes whose node.roles setting does not contain "data_frozen" or "data") will use a
* value of 0 for the setting.
*/
public static ByteSizeValue getSnapshotCacheSize(Settings settings) {
List<DiscoveryNodeRole> nodeRoles = NodeRoleSettings.NODE_ROLES_SETTING.get(settings);
if (DataTier.isFrozenNode(Set.of(nodeRoles.toArray(DiscoveryNodeRole[]::new)))) {
return SNAPSHOT_CACHE_SIZE_SETTING.get(settings);
} else {
return ByteSizeValue.ZERO;
}
}

public static final String SNAPSHOT_CACHE_SIZE = SHARED_CACHE_SETTINGS_PREFIX + "size";
public static final Setting<ByteSizeValue> SNAPSHOT_CACHE_SIZE_SETTING = new Setting<>(
SHARED_CACHE_SETTINGS_PREFIX + "size",
SNAPSHOT_CACHE_SIZE,
ByteSizeValue.ZERO.getStringRep(),
s -> ByteSizeValue.parseBytesSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "size"),
s -> ByteSizeValue.parseBytesSizeValue(s, SNAPSHOT_CACHE_SIZE),
new Setting.Validator<ByteSizeValue>() {

@Override
Expand All @@ -105,7 +120,7 @@ public void validate(final ByteSizeValue value) {
@Override
public void validate(final ByteSizeValue value, final Map<Setting<?>, Object> settings) {
if (value.getBytes() == -1) {
throw new SettingsException("setting [{}] must be non-negative", SHARED_CACHE_SETTINGS_PREFIX + "size");
throw new SettingsException("setting [{}] must be non-negative", SNAPSHOT_CACHE_SIZE);
}
if (value.getBytes() > 0) {
@SuppressWarnings("unchecked")
Expand All @@ -115,7 +130,7 @@ public void validate(final ByteSizeValue value, final Map<Setting<?>, Object> se
DeprecationCategory.SETTINGS,
"shared_cache",
"setting [{}] to be positive [{}] on node without the data_frozen role is deprecated, roles are [{}]",
SHARED_CACHE_SETTINGS_PREFIX + "size",
SNAPSHOT_CACHE_SIZE,
value.getStringRep(),
roles.stream().map(DiscoveryNodeRole::roleName).collect(Collectors.joining(","))
);
Expand Down Expand Up @@ -190,7 +205,7 @@ public Iterator<Setting<?>> settings() {
@SuppressWarnings({ "unchecked", "rawtypes" })
public FrozenCacheService(NodeEnvironment environment, Settings settings, ThreadPool threadPool) {
this.currentTimeSupplier = threadPool::relativeTimeInMillis;
final long cacheSize = SNAPSHOT_CACHE_SIZE_SETTING.get(settings).getBytes();
final long cacheSize = getSnapshotCacheSize(settings).getBytes();
final long regionSize = SNAPSHOT_CACHE_REGION_SIZE_SETTING.get(settings).getBytes();
final int numRegions = Math.toIntExact(cacheSize / regionSize);
keyMapping = new ConcurrentHashMap<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ protected FrozenCacheService defaultFrozenCacheService() {
protected FrozenCacheService randomFrozenCacheService() {
final Settings.Builder cacheSettings = Settings.builder();
if (randomBoolean()) {
cacheSettings.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), randomFrozenCacheSize());
cacheSettings.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, randomFrozenCacheSize());
}
if (randomBoolean()) {
cacheSettings.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), pageAligned(randomFrozenCacheSize()));
Expand Down Expand Up @@ -185,7 +185,7 @@ protected FrozenCacheService createFrozenCacheService(final ByteSizeValue cacheS
return new FrozenCacheService(
nodeEnvironment,
Settings.builder()
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize)
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, cacheSize)
.put(FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), cacheRangeSize)
.build(),
threadPool
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.Set;

import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
import static org.hamcrest.Matchers.equalTo;

public class FrozenCacheServiceTests extends ESTestCase {

Expand All @@ -34,11 +35,15 @@ private static long size(long numPages) {
}

public void testBasicEviction() throws IOException {
DiscoveryNode.setAdditionalRoles(
Set.of(DataTier.DATA_HOT_NODE_ROLE, DataTier.DATA_WARM_NODE_ROLE, DataTier.DATA_COLD_NODE_ROLE, DataTier.DATA_FROZEN_NODE_ROLE)
);
Settings settings = Settings.builder()
.put(NODE_NAME_SETTING.getKey(), "node")
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(size(500)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, new ByteSizeValue(size(500)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), new ByteSizeValue(size(100)).getStringRep())
.put("path.home", createTempDir())
.putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DataTier.DATA_FROZEN_NODE_ROLE.roleName())
.build();
final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random());
try (
Expand Down Expand Up @@ -79,11 +84,15 @@ public void testBasicEviction() throws IOException {
}

public void testAutoEviction() throws IOException {
DiscoveryNode.setAdditionalRoles(
Set.of(DataTier.DATA_HOT_NODE_ROLE, DataTier.DATA_WARM_NODE_ROLE, DataTier.DATA_COLD_NODE_ROLE, DataTier.DATA_FROZEN_NODE_ROLE)
);
Settings settings = Settings.builder()
.put(NODE_NAME_SETTING.getKey(), "node")
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(size(200)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, new ByteSizeValue(size(200)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), new ByteSizeValue(size(100)).getStringRep())
.put("path.home", createTempDir())
.putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DataTier.DATA_FROZEN_NODE_ROLE.roleName())
.build();
final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random());
try (
Expand Down Expand Up @@ -115,11 +124,15 @@ public void testAutoEviction() throws IOException {
}

public void testForceEviction() throws IOException {
DiscoveryNode.setAdditionalRoles(
Set.of(DataTier.DATA_HOT_NODE_ROLE, DataTier.DATA_WARM_NODE_ROLE, DataTier.DATA_COLD_NODE_ROLE, DataTier.DATA_FROZEN_NODE_ROLE)
);
Settings settings = Settings.builder()
.put(NODE_NAME_SETTING.getKey(), "node")
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(size(500)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, new ByteSizeValue(size(500)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), new ByteSizeValue(size(100)).getStringRep())
.put("path.home", createTempDir())
.putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DataTier.DATA_FROZEN_NODE_ROLE.roleName())
.build();
final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random());
try (
Expand All @@ -143,11 +156,15 @@ public void testForceEviction() throws IOException {
}

public void testDecay() throws IOException {
DiscoveryNode.setAdditionalRoles(
Set.of(DataTier.DATA_HOT_NODE_ROLE, DataTier.DATA_WARM_NODE_ROLE, DataTier.DATA_COLD_NODE_ROLE, DataTier.DATA_FROZEN_NODE_ROLE)
);
Settings settings = Settings.builder()
.put(NODE_NAME_SETTING.getKey(), "node")
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(size(500)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, new ByteSizeValue(size(500)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), new ByteSizeValue(size(100)).getStringRep())
.put("path.home", createTempDir())
.putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DataTier.DATA_FROZEN_NODE_ROLE.roleName())
.build();
final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random());
try (
Expand Down Expand Up @@ -201,20 +218,42 @@ public void testCacheSizeDeprecatedOnNonFrozenNodes() {
Set.of(DataTier.DATA_HOT_NODE_ROLE, DataTier.DATA_WARM_NODE_ROLE, DataTier.DATA_COLD_NODE_ROLE, DataTier.DATA_FROZEN_NODE_ROLE)
);
final Settings settings = Settings.builder()
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(size(500)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, new ByteSizeValue(size(500)).getStringRep())
.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), new ByteSizeValue(size(100)).getStringRep())
.putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DataTier.DATA_HOT_NODE_ROLE.roleName())
.build();
FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.get(settings);
assertWarnings(
"setting ["
+ FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey()
+ FrozenCacheService.SNAPSHOT_CACHE_SIZE
+ "] to be positive ["
+ new ByteSizeValue(size(500)).getStringRep()
+ "] on node without the data_frozen role is deprecated, roles are [data_hot]"
);
}

public void testCacheSizeZeroOnNonFrozenNodes() {
DiscoveryNode.setAdditionalRoles(
Set.of(DataTier.DATA_HOT_NODE_ROLE, DataTier.DATA_WARM_NODE_ROLE, DataTier.DATA_COLD_NODE_ROLE, DataTier.DATA_FROZEN_NODE_ROLE)
);
{
final Settings settings = Settings.builder()
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, "500b")
.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b")
.putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DataTier.DATA_HOT_NODE_ROLE.roleName())
.build();
assertThat(FrozenCacheService.getSnapshotCacheSize(settings), equalTo(ByteSizeValue.ZERO));
}
{
final Settings settings = Settings.builder()
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, "500b")
.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b")
.putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DataTier.DATA_FROZEN_NODE_ROLE.roleName())
.build();
assertThat(FrozenCacheService.getSnapshotCacheSize(settings), equalTo(ByteSizeValue.ofBytes(500)));
}
}

private static CacheKey generateCacheKey() {
return new CacheKey(
randomAlphaOfLength(10),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public void testRandomReads() throws IOException {
final Settings settings = Settings.builder()
.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), regionSize)
.put(FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), rangeSize)
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize)
.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE, cacheSize)
.put("path.home", createTempDir())
.build();
final Environment environment = TestEnvironment.newEnvironment(settings);
Expand Down