From a9454464b233218a90f85ee12be63beac1f9d66e Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 31 Oct 2023 13:34:37 +0200 Subject: [PATCH 01/12] Refactor: create new dedicated package for data tiers --- .../routing/allocation/DataTierAllocationDeciderIT.java | 2 +- x-pack/plugin/core/src/main/java/module-info.java | 1 + .../java/org/elasticsearch/xpack/core/XPackClientPlugin.java | 1 + .../main/java/org/elasticsearch/xpack/core/XPackPlugin.java | 2 ++ .../xpack/core/{ => datatiers}/DataTiersFeatureSetUsage.java | 4 +++- .../core/{ => datatiers}/DataTiersInfoTransportAction.java | 3 ++- .../core/{ => datatiers}/DataTiersUsageTransportAction.java | 2 +- .../core/{ => datatiers}/DataTiersFeatureSetUsageTests.java | 2 +- .../{ => datatiers}/DataTiersUsageTransportActionTests.java | 2 +- 9 files changed, 13 insertions(+), 6 deletions(-) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{ => datatiers}/DataTiersFeatureSetUsage.java (98%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{ => datatiers}/DataTiersInfoTransportAction.java (91%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{ => datatiers}/DataTiersUsageTransportAction.java (99%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/{ => datatiers}/DataTiersFeatureSetUsageTests.java (97%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/{ => datatiers}/DataTiersUsageTransportActionTests.java (99%) diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java index 7124db6c1c721..cbceb320dfa04 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java @@ -26,9 +26,9 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.core.DataTiersFeatureSetUsage; import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder; import org.elasticsearch.xpack.core.action.XPackUsageResponse; +import org.elasticsearch.xpack.core.datatiers.DataTiersFeatureSetUsage; import org.junit.Before; import java.util.ArrayList; diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index deb3c4384a04b..d7b5a86d87f90 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -57,6 +57,7 @@ exports org.elasticsearch.xpack.core.common.validation; exports org.elasticsearch.xpack.core.common; exports org.elasticsearch.xpack.core.datastreams; + exports org.elasticsearch.xpack.core.datatiers; exports org.elasticsearch.xpack.core.deprecation; exports org.elasticsearch.xpack.core.downsample; exports org.elasticsearch.xpack.core.enrich.action; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 6d019e50f9d5f..ac16631bacb73 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage; import org.elasticsearch.xpack.core.datastreams.DataStreamLifecycleFeatureSetUsage; +import org.elasticsearch.xpack.core.datatiers.DataTiersFeatureSetUsage; import org.elasticsearch.xpack.core.downsample.DownsampleShardStatus; import org.elasticsearch.xpack.core.enrich.EnrichFeatureSetUsage; import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyStatus; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index d02e3f43d80cb..d8ac52409db10 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -98,6 +98,8 @@ import org.elasticsearch.xpack.core.action.XPackUsageResponse; import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.async.TransportDeleteAsyncResultAction; +import org.elasticsearch.xpack.core.datatiers.DataTiersInfoTransportAction; +import org.elasticsearch.xpack.core.datatiers.DataTiersUsageTransportAction; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction; import org.elasticsearch.xpack.core.rest.action.RestXPackUsageAction; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsage.java similarity index 98% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsage.java index 0bf21f66b4888..f990118763bad 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsage.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core; +package org.elasticsearch.xpack.core.datatiers; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -16,6 +16,8 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; import java.io.IOException; import java.util.Collections; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersInfoTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java similarity index 91% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersInfoTransportAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java index 6134813dc4651..3af1945c53d3f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersInfoTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java @@ -5,11 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.core; +package org.elasticsearch.xpack.core.datatiers; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java similarity index 99% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java index 295df1ea51b6b..1381fdd3946d8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core; +package org.elasticsearch.xpack.core.datatiers; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsageTests.java similarity index 97% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsageTests.java index e5f37dfb5764c..0951408441b3f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsageTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core; +package org.elasticsearch.xpack.core.datatiers; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.io.stream.Writeable; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java similarity index 99% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersUsageTransportActionTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java index 93e991b0fa5ae..40ad14245d152 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersUsageTransportActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core; +package org.elasticsearch.xpack.core.datatiers; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStats; From 722f6b1c8179d819e1433728462133a9ed55d984 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 31 Oct 2023 15:03:36 +0200 Subject: [PATCH 02/12] Introduce new transport action to calculate local stats --- .../elasticsearch/xpack/core/XPackPlugin.java | 2 + .../core/datatiers/NodeDataTiersUsage.java | 133 ++++++++ .../NodesDataTiersUsageTransportAction.java | 239 +++++++++++++ ...desDataTiersUsageTransportActionTests.java | 322 ++++++++++++++++++ .../xpack/security/operator/Constants.java | 1 + 5 files changed, 697 insertions(+) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index d8ac52409db10..66534cccff064 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -100,6 +100,7 @@ import org.elasticsearch.xpack.core.async.TransportDeleteAsyncResultAction; import org.elasticsearch.xpack.core.datatiers.DataTiersInfoTransportAction; import org.elasticsearch.xpack.core.datatiers.DataTiersUsageTransportAction; +import org.elasticsearch.xpack.core.datatiers.NodesDataTiersUsageTransportAction; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction; import org.elasticsearch.xpack.core.rest.action.RestXPackUsageAction; @@ -364,6 +365,7 @@ public Collection createComponents(PluginServices services) { actions.add(new ActionHandler<>(XPackUsageFeatureAction.DATA_STREAM_LIFECYCLE, DataStreamLifecycleUsageTransportAction.class)); actions.add(new ActionHandler<>(XPackUsageFeatureAction.HEALTH, HealthApiUsageTransportAction.class)); actions.add(new ActionHandler<>(XPackUsageFeatureAction.REMOTE_CLUSTERS, RemoteClusterUsageTransportAction.class)); + actions.add(new ActionHandler<>(NodesDataTiersUsageTransportAction.TYPE, NodesDataTiersUsageTransportAction.class)); return actions; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java new file mode 100644 index 0000000000000..d99bf19bcf586 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Data tier usage statistics on a specific node. The statistics groups the indices, shard sizes, shard counts based + * on their tier preference. + */ +public class NodeDataTiersUsage extends BaseNodeResponse { + + private final Map usageStatsByTier; + + public static class UsageStats implements Writeable { + private final Set indices; + private final List primaryShardSizes; + private int totalShardCount; + private long docCount; + private long totalSize; + + public UsageStats() { + this.indices = new HashSet<>(); + this.primaryShardSizes = new ArrayList<>(); + this.totalShardCount = 0; + this.docCount = 0; + this.totalSize = 0; + } + + public UsageStats(Set indices, List primaryShardSizes, int totalShardCount, long docCount, long totalSize) { + this.indices = indices; + this.primaryShardSizes = primaryShardSizes; + this.totalShardCount = totalShardCount; + this.docCount = docCount; + this.totalSize = totalSize; + } + + static UsageStats read(StreamInput in) throws IOException { + return new UsageStats( + in.readCollectionAsSet(StreamInput::readString), + in.readCollectionAsList(StreamInput::readVLong), + in.readVInt(), + in.readVLong(), + in.readVLong() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(indices, StreamOutput::writeString); + out.writeCollection(primaryShardSizes, StreamOutput::writeVLong); + out.writeVInt(totalShardCount); + out.writeVLong(docCount); + out.writeVLong(totalSize); + } + + public void addIndex(String indexName) { + indices.add(indexName); + } + + public void addPrimaryShardSize(long primaryShardSize) { + primaryShardSizes.add(primaryShardSize); + } + + public void incrementTotalSize(long totalSize) { + this.totalSize += totalSize; + } + + public void incrementDocCount(long docCount) { + this.docCount += docCount; + } + + public void incrementTotalShardCount(int totalShardCount) { + this.totalShardCount += totalShardCount; + } + + public Set getIndices() { + return indices; + } + + public List getPrimaryShardSizes() { + return primaryShardSizes; + } + + public int getTotalShardCount() { + return totalShardCount; + } + + public long getDocCount() { + return docCount; + } + + public long getTotalSize() { + return totalSize; + } + } + + public NodeDataTiersUsage(StreamInput in) throws IOException { + super(in); + usageStatsByTier = in.readMap(UsageStats::read); + } + + public NodeDataTiersUsage(DiscoveryNode node, Map usageStatsByTier) { + super(node); + this.usageStatsByTier = usageStatsByTier; + } + + public Map getUsageStatsByTier() { + return Map.copyOf(usageStatsByTier); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeMap(usageStatsByTier, (o, v) -> v.writeTo(o)); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java new file mode 100644 index 0000000000000..2a3787a6e5acb --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Sources locally data tier usage stats mainly indices and shard sizes grouped by preferred data tier. + */ +public class NodesDataTiersUsageTransportAction extends TransportNodesAction< + NodesDataTiersUsageTransportAction.NodesRequest, + NodesDataTiersUsageTransportAction.NodesResponse, + NodesDataTiersUsageTransportAction.NodeRequest, + NodeDataTiersUsage> { + + public static final ActionType TYPE = ActionType.localOnly("cluster:monitor/nodes/data_tier_usage"); + + private final IndicesService indicesService; + + @Inject + public NodesDataTiersUsageTransportAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + IndicesService indicesService, + ActionFilters actionFilters + ) { + super( + TYPE.name(), + clusterService, + transportService, + actionFilters, + NodeRequest::new, + threadPool.executor(ThreadPool.Names.MANAGEMENT) + ); + this.indicesService = indicesService; + } + + @Override + protected NodesResponse newResponse(NodesRequest request, List responses, List failures) { + return new NodesResponse(clusterService.getClusterName(), responses, failures); + } + + @Override + protected NodeRequest newNodeRequest(NodesRequest request) { + return NodeRequest.INSTANCE; + } + + @Override + protected NodeDataTiersUsage newNodeResponse(StreamInput in, DiscoveryNode node) throws IOException { + return new NodeDataTiersUsage(in); + } + + @Override + protected NodeDataTiersUsage nodeOperation(NodeRequest nodeRequest, Task task) { + assert task instanceof CancellableTask; + + DiscoveryNode localNode = clusterService.localNode(); + CommonStatsFlags flags = CommonStatsFlags.NONE.set(CommonStatsFlags.Flag.Docs, true).set(CommonStatsFlags.Flag.Store, true); + NodeIndicesStats nodeIndicesStats = indicesService.stats(flags, true); + ClusterState state = clusterService.state(); + RoutingNode routingNode = state.getRoutingNodes().node(localNode.getId()); + Map usageStatsByTier = aggregateStats(routingNode, state.metadata(), nodeIndicesStats); + return new NodeDataTiersUsage(clusterService.localNode(), usageStatsByTier); + } + + // For testing + static Map aggregateStats( + RoutingNode routingNode, + Metadata metadata, + NodeIndicesStats nodeIndicesStats + ) { + if (routingNode == null) { + return Map.of(); + } + Map usageStatsByTier = new HashMap<>(); + Set localIndices = StreamSupport.stream(routingNode.spliterator(), false) + .map(routing -> routing.index().getName()) + .collect(Collectors.toSet()); + for (String indexName : localIndices) { + IndexMetadata indexMetadata = metadata.index(indexName); + String tier = findPreferredTier(indexMetadata); + if (tier != null) { + NodeDataTiersUsage.UsageStats usageStats = usageStatsByTier.computeIfAbsent( + tier, + ignored -> new NodeDataTiersUsage.UsageStats() + ); + usageStats.addIndex(indexName); + List allShardStats = nodeIndicesStats.getShardStats(indexMetadata.getIndex()); + if (allShardStats != null) { + for (IndexShardStats indexShardStats : allShardStats) { + usageStats.incrementTotalSize(indexShardStats.getTotal().getStore().totalDataSetSizeInBytes()); + usageStats.incrementDocCount(indexShardStats.getTotal().getDocs().getCount()); + + ShardRouting shardRouting = routingNode.getByShardId(indexShardStats.getShardId()); + if (shardRouting != null && shardRouting.state() == ShardRoutingState.STARTED) { + usageStats.incrementTotalShardCount(1); + + // Accumulate stats about started primary shards + StoreStats primaryStoreStats = indexShardStats.getPrimary().getStore(); + if (shardRouting.primary() && primaryStoreStats != null) { + usageStats.addPrimaryShardSize(primaryStoreStats.totalDataSetSizeInBytes()); + } + } + } + } + } + } + return usageStatsByTier; + } + + @Nullable + static String findPreferredTier(IndexMetadata indexMetadata) { + List tierPref = DataTier.parseTierList(indexMetadata.getSettings().get(DataTier.TIER_PREFERENCE)) + .stream() + .filter(DataTier::validTierName) + .toList(); + if (tierPref.isEmpty() == false) { + return tierPref.get(0); + } + return null; + } + + public static class NodesRequest extends BaseNodesRequest { + + public NodesRequest() { + super((String[]) null); + } + + public NodesRequest(StreamInput in) throws IOException { + super(in); + } + + /** + * Get stats from nodes based on the nodes ids specified. If none are passed, stats + * for all nodes will be returned. + */ + public NodesRequest(String... nodesIds) { + super(nodesIds); + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + } + + public static class NodeRequest extends TransportRequest { + + static final NodeRequest INSTANCE = new NodeRequest(); + + public NodeRequest(StreamInput in) throws IOException { + super(in); + } + + public NodeRequest() { + + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + } + + public static class NodesResponse extends BaseNodesResponse { + + public NodesResponse(StreamInput in) throws IOException { + super(in); + } + + public NodesResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readCollectionAsList(NodeDataTiersUsage::new); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeCollection(nodes); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java new file mode 100644 index 0000000000000..dff234c3c832d --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java @@ -0,0 +1,322 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.IndexLongFieldRange; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class NodesDataTiersUsageTransportActionTests extends ESTestCase { + + private static final CommonStats COMMON_STATS = new CommonStats( + CommonStatsFlags.NONE.set(CommonStatsFlags.Flag.Docs, true).set(CommonStatsFlags.Flag.Store, true) + ); + private long byteSize; + private long docCount; + + @Before + public void setup() { + byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB + docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million + } + + public void testTierIndices() { + IndexMetadata hotIndex1 = indexMetadata("hot-1", 1, 0, DataTier.DATA_HOT); + IndexMetadata hotIndex2 = indexMetadata("hot-2", 1, 0, DataTier.DATA_HOT); + IndexMetadata warmIndex1 = indexMetadata("warm-1", 1, 0, DataTier.DATA_WARM); + IndexMetadata coldIndex1 = indexMetadata("cold-1", 1, 0, DataTier.DATA_COLD); + IndexMetadata coldIndex2 = indexMetadata("cold-2", 1, 0, DataTier.DATA_COLD, DataTier.DATA_WARM); // Prefers cold over warm + IndexMetadata nonTiered = indexMetadata("non-tier", 1, 0); // No tier + IndexMetadata invalidTiered = indexMetadata("non-tier", 1, 0, "invalid-tier"); // No tier + + assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(hotIndex1), equalTo(DataTier.DATA_HOT)); + assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(hotIndex2), equalTo(DataTier.DATA_HOT)); + assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(warmIndex1), equalTo(DataTier.DATA_WARM)); + assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(coldIndex1), equalTo(DataTier.DATA_COLD)); + assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(coldIndex2), equalTo(DataTier.DATA_COLD)); + assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(nonTiered), nullValue()); + assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(invalidTiered), nullValue()); + } + + public void testCalculateStatsNoTiers() { + // Nodes: 0 Tiered Nodes, 1 Data Node + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_ROLE); + discoBuilder.add(dataNode1); + discoBuilder.localNodeId(dataNode1.getId()); + + // Indices: 1 Regular index + Metadata.Builder metadataBuilder = Metadata.builder(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + + IndexMetadata index1 = indexMetadata("index_1", 3, 1); + metadataBuilder.put(index1, false).generateClusterUuidIfNeeded(); + + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index1.getIndex()); + routeTestShardToNodes(index1, 0, indexRoutingTableBuilder, dataNode1); + routeTestShardToNodes(index1, 1, indexRoutingTableBuilder, dataNode1); + routeTestShardToNodes(index1, 2, indexRoutingTableBuilder, dataNode1); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + + // Cluster State and create stats responses + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(metadataBuilder) + .nodes(discoBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + NodeIndicesStats nodeIndicesStats = buildNodeIndicesStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + byteSize, + docCount + ); + + // Calculate usage + Map usageStats = NodesDataTiersUsageTransportAction.aggregateStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + clusterState.metadata(), + nodeIndicesStats + ); + + // Verify - No results when no tiers present + assertThat(usageStats.size(), is(0)); + } + + public void testCalculateStatsNoIndices() { + // Nodes: 1 Data, 1 Hot, 1 Warm, 1 Cold, 1 Frozen + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); + discoBuilder.add(dataNode1); + discoBuilder.localNodeId(dataNode1.getId()); + + // Indices: 1 Regular index, not hosted on any tiers + Metadata.Builder metadataBuilder = Metadata.builder(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + + // Cluster State and create stats responses + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(metadataBuilder) + .nodes(discoBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + NodeIndicesStats nodeIndicesStats = buildNodeIndicesStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + byteSize, + docCount + ); + + // Calculate usage + Map usageStats = NodesDataTiersUsageTransportAction.aggregateStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + clusterState.metadata(), + nodeIndicesStats + ); + + // Verify - No results when no tiers present + assertThat(usageStats.size(), is(0)); + } + + public void testCalculateStatsTieredIndicesOnly() { + // Nodes: 3 Data, 0 Tiered - Only hosting indices on generic data nodes + int nodeId = 0; + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + + DiscoveryNode dataNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); + discoBuilder.add(dataNode1); + DiscoveryNode dataNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); + discoBuilder.add(dataNode2); + + discoBuilder.localNodeId(dataNode1.getId()); + + // Indices: 1 Hot index, 2 Warm indices, 3 Cold indices + Metadata.Builder metadataBuilder = Metadata.builder(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + + IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); + metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); + { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); + routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, dataNode1, dataNode2); + routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, dataNode2, dataNode1); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + } + + IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM); + metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); + { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); + routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, dataNode1, dataNode2); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + } + IndexMetadata warmIndex2 = indexMetadata("warm_index_2", 1, 1, DataTier.DATA_WARM); + metadataBuilder.put(warmIndex2, false).generateClusterUuidIfNeeded(); + { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex2.getIndex()); + routeTestShardToNodes(warmIndex2, 0, indexRoutingTableBuilder, dataNode2, dataNode1); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + } + + IndexMetadata coldIndex1 = indexMetadata("cold_index_1", 1, 0, DataTier.DATA_COLD); + metadataBuilder.put(coldIndex1, false).generateClusterUuidIfNeeded(); + { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex1.getIndex()); + routeTestShardToNodes(coldIndex1, 0, indexRoutingTableBuilder, dataNode1); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + } + + // Cluster State and create stats responses + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .nodes(discoBuilder) + .metadata(metadataBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + NodeIndicesStats nodeIndicesStats = buildNodeIndicesStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + byteSize, + docCount + ); + + // Calculate usage + Map usageStats = NodesDataTiersUsageTransportAction.aggregateStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + clusterState.metadata(), + nodeIndicesStats + ); + + // Verify - Index stats exist for the tiers, but no tiered nodes are found + assertThat(usageStats.size(), is(3)); + + NodeDataTiersUsage.UsageStats hotStats = usageStats.get(DataTier.DATA_HOT); + assertThat(hotStats, is(notNullValue())); + assertThat(hotStats.getIndices(), equalTo(Set.of(hotIndex1.getIndex().getName()))); + assertThat(hotStats.getPrimaryShardSizes(), equalTo(List.of(byteSize))); + assertThat(hotStats.getTotalShardCount(), is(2)); + assertThat(hotStats.getDocCount(), is(hotStats.getTotalShardCount() * docCount)); + assertThat(hotStats.getTotalSize(), is(hotStats.getTotalShardCount() * byteSize)); + + NodeDataTiersUsage.UsageStats warmStats = usageStats.get(DataTier.DATA_WARM); + assertThat(warmStats, is(notNullValue())); + assertThat(warmStats.getIndices(), equalTo(Set.of(warmIndex1.getIndex().getName(), warmIndex2.getIndex().getName()))); + assertThat(warmStats.getPrimaryShardSizes(), equalTo(List.of(byteSize))); + assertThat(warmStats.getTotalShardCount(), is(2)); + assertThat(warmStats.getDocCount(), is(warmStats.getTotalShardCount() * docCount)); + assertThat(warmStats.getTotalSize(), is(warmStats.getTotalShardCount() * byteSize)); + + NodeDataTiersUsage.UsageStats coldStats = usageStats.get(DataTier.DATA_COLD); + assertThat(coldStats, is(notNullValue())); + assertThat(coldStats.getIndices(), equalTo(Set.of(coldIndex1.getIndex().getName()))); + assertThat(coldStats.getPrimaryShardSizes(), equalTo(List.of(byteSize))); + assertThat(coldStats.getTotalShardCount(), is(1)); + assertThat(coldStats.getDocCount(), is(coldStats.getTotalShardCount() * docCount)); + assertThat(coldStats.getTotalSize(), is(coldStats.getTotalShardCount() * byteSize)); + } + + private static DiscoveryNode newNode(int nodeId, DiscoveryNodeRole... roles) { + return DiscoveryNodeUtils.builder("node_" + nodeId).roles(Set.of(roles)).build(); + } + + private static void routeTestShardToNodes( + IndexMetadata index, + int shard, + IndexRoutingTable.Builder indexRoutingTableBuilder, + DiscoveryNode... nodes + ) { + ShardId shardId = new ShardId(index.getIndex(), shard); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); + boolean primary = true; + for (DiscoveryNode node : nodes) { + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(shardId, node.getId(), null, primary, ShardRoutingState.STARTED) + ); + primary = false; + } + indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder); + } + + private NodeIndicesStats buildNodeIndicesStats(RoutingNode routingNode, long bytesPerShard, long docsPerShard) { + Map> indexStats = new HashMap<>(); + for (ShardRouting shardRouting : routingNode) { + ShardId shardId = shardRouting.shardId(); + ShardStats shardStat = shardStat(bytesPerShard, docsPerShard, shardRouting); + IndexShardStats shardStats = new IndexShardStats(shardId, new ShardStats[] { shardStat }); + indexStats.computeIfAbsent(shardId.getIndex(), k -> new ArrayList<>()).add(shardStats); + } + return new NodeIndicesStats(COMMON_STATS, Map.of(), indexStats, true); + } + + private static ShardStats shardStat(long byteCount, long docCount, ShardRouting routing) { + StoreStats storeStats = new StoreStats(randomNonNegativeLong(), byteCount, 0L); + DocsStats docsStats = new DocsStats(docCount, 0L, byteCount); + Path fakePath = PathUtils.get("test/dir/" + routing.shardId().getIndex().getUUID() + "/" + routing.shardId().id()); + ShardPath fakeShardPath = new ShardPath(false, fakePath, fakePath, routing.shardId()); + CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL); + commonStats.getStore().add(storeStats); + commonStats.getDocs().add(docsStats); + return new ShardStats(routing, fakeShardPath, commonStats, null, null, null, false, 0); + } + + private static IndexMetadata indexMetadata(String indexName, int numberOfShards, int numberOfReplicas, String... dataTierPrefs) { + Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), numberOfShards, numberOfReplicas).put( + SETTING_CREATION_DATE, + System.currentTimeMillis() + ); + + if (dataTierPrefs.length > 1) { + StringBuilder tierBuilder = new StringBuilder(dataTierPrefs[0]); + for (int idx = 1; idx < dataTierPrefs.length; idx++) { + tierBuilder.append(',').append(dataTierPrefs[idx]); + } + settingsBuilder.put(DataTier.TIER_PREFERENCE, tierBuilder.toString()); + } else if (dataTierPrefs.length == 1) { + settingsBuilder.put(DataTier.TIER_PREFERENCE, dataTierPrefs[0]); + } + + return IndexMetadata.builder(indexName).settings(settingsBuilder.build()).timestampRange(IndexLongFieldRange.UNKNOWN).build(); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 9f490792d800f..fc5f5ba616ab8 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -302,6 +302,7 @@ public class Constants { "cluster:monitor/update/health/info", "cluster:monitor/ingest/geoip/stats", "cluster:monitor/main", + "cluster:monitor/nodes/data_tier_usage", "cluster:monitor/nodes/hot_threads", "cluster:monitor/nodes/info", "cluster:monitor/nodes/stats", From fb7397f1c5224dc2736014a28afdf2e6202db410 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 31 Oct 2023 20:23:47 +0200 Subject: [PATCH 03/12] Use the new transport action in DataTiersUsageTransportAction --- .../DataTiersUsageRestCancellationIT.java | 9 +- .../DataTiersUsageTransportAction.java | 151 +--- .../DataTiersUsageTransportActionTests.java | 656 +++++------------- 3 files changed, 218 insertions(+), 598 deletions(-) diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java index f669bb8589fd7..faeb760b3c181 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.core.rest.action; import org.apache.http.client.methods.HttpGet; -import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; @@ -35,6 +34,7 @@ import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageResponse; +import org.elasticsearch.xpack.core.datatiers.NodesDataTiersUsageTransportAction; import java.nio.file.Path; import java.util.Arrays; @@ -76,7 +76,7 @@ public void testCancellation() throws Exception { final SubscribableListener nodeStatsRequestsReleaseListener = new SubscribableListener<>(); for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { ((MockTransportService) transportService).addRequestHandlingBehavior( - TransportNodesStatsAction.TYPE.name() + "[n]", + NodesDataTiersUsageTransportAction.TYPE.name() + "[n]", (handler, request, channel, task) -> { tasksBlockedLatch.countDown(); nodeStatsRequestsReleaseListener.addListener( @@ -94,14 +94,13 @@ public void testCancellation() throws Exception { safeAwait(tasksBlockedLatch); // must wait for the node-level tasks to start to avoid cancelling being handled earlier cancellable.cancel(); - // NB this test works by blocking node-level stats requests; when #100230 is addressed this will need to target a different action. - assertAllCancellableTasksAreCancelled(TransportNodesStatsAction.TYPE.name()); + assertAllCancellableTasksAreCancelled(NodesDataTiersUsageTransportAction.TYPE.name()); assertAllCancellableTasksAreCancelled(XPackUsageAction.NAME); nodeStatsRequestsReleaseListener.onResponse(null); expectThrows(CancellationException.class, future::actionGet); - assertAllTasksHaveFinished(TransportNodesStatsAction.TYPE.name()); + assertAllTasksHaveFinished(NodesDataTiersUsageTransportAction.TYPE.name()); assertAllTasksHaveFinished(XPackUsageAction.NAME); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java index 1381fdd3946d8..97b1b44312ab1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java @@ -8,26 +8,15 @@ package org.elasticsearch.xpack.core.datatiers; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.protocol.xpack.XPackUsageRequest; import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.tasks.Task; @@ -42,7 +31,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.StreamSupport; public class DataTiersUsageTransportAction extends XPackUsageFeatureTransportAction { @@ -77,41 +65,14 @@ protected void masterOperation( ) { new ParentTaskAssigningClient(client, clusterService.localNode(), task).admin() .cluster() - .prepareNodesStats() - .all() - .setIndices(CommonStatsFlags.ALL) - .execute(listener.delegateFailureAndWrap((delegate, nodesStatsResponse) -> { - final RoutingNodes routingNodes = state.getRoutingNodes(); - final Map indices = state.getMetadata().getIndices(); - - // Determine which tiers each index would prefer to be within - Map indicesToTiers = tierIndices(indices); - - // Generate tier specific stats for the nodes and indices - Map tierSpecificStats = calculateStats( - nodesStatsResponse.getNodes(), - indicesToTiers, - routingNodes - ); - - delegate.onResponse(new XPackUsageFeatureResponse(new DataTiersFeatureSetUsage(tierSpecificStats))); - })); - } - - // Visible for testing - // Takes a registry of indices and returns a mapping of index name to which tier it most prefers. Always 1 to 1, some may filter out. - static Map tierIndices(Map indices) { - Map indexByTier = new HashMap<>(); - indices.entrySet().forEach(entry -> { - String tierPref = entry.getValue().getSettings().get(DataTier.TIER_PREFERENCE); - if (Strings.hasText(tierPref)) { - String[] tiers = tierPref.split(","); - if (tiers.length > 0) { - indexByTier.put(entry.getKey(), tiers[0]); - } - } - }); - return indexByTier; + .execute( + NodesDataTiersUsageTransportAction.TYPE, + new NodesDataTiersUsageTransportAction.NodesRequest(), + listener.delegateFailureAndWrap((delegate, response) -> { + // Generate tier specific stats for the nodes and indices + delegate.onResponse(new XPackUsageFeatureResponse(new DataTiersFeatureSetUsage(aggregateStats(response.getNodes())))); + }) + ); } /** @@ -129,27 +90,24 @@ private static class TierStatsAccumulator { } // Visible for testing - static Map calculateStats( - List nodesStats, - Map indexByTier, - RoutingNodes routingNodes - ) { + static Map aggregateStats(List nodeDataTiersUsages) { Map statsAccumulators = new HashMap<>(); - for (NodeStats nodeStats : nodesStats) { - aggregateDataTierNodeCounts(nodeStats, statsAccumulators); - aggregateDataTierIndexStats(nodeStats, routingNodes, indexByTier, statsAccumulators); + for (NodeDataTiersUsage nodeDataTiersUsage : nodeDataTiersUsages) { + // This ensures we only count the nodes that responded + aggregateDataTierNodeCounts(nodeDataTiersUsage, statsAccumulators); + aggregateDataTierIndexStats(nodeDataTiersUsage, statsAccumulators); } Map results = new HashMap<>(); for (Map.Entry entry : statsAccumulators.entrySet()) { - results.put(entry.getKey(), calculateFinalTierStats(entry.getValue())); + results.put(entry.getKey(), aggregateFinalTierStats(entry.getValue())); } return results; } /** - * Determine which data tiers this node belongs to (if any), and increment the node counts for those tiers. + * Determine which data tiers each node belongs to (if any), and increment the node counts for those tiers. */ - private static void aggregateDataTierNodeCounts(NodeStats nodeStats, Map tiersStats) { + private static void aggregateDataTierNodeCounts(NodeDataTiersUsage nodeStats, Map tiersStats) { nodeStats.getNode() .getRoles() .stream() @@ -159,73 +117,28 @@ private static void aggregateDataTierNodeCounts(NodeStats nodeStats, Map indexByTier, - Map accumulators - ) { - final RoutingNode node = routingNodes.node(nodeStats.getNode().getId()); - if (node != null) { - StreamSupport.stream(node.spliterator(), false) - .map(ShardRouting::index) - .distinct() - .forEach(index -> classifyIndexAndCollectStats(index, nodeStats, indexByTier, node, accumulators)); - } - } - - /** - * Determine which tier an index belongs in, then accumulate its stats into that tier's stats. - */ - private static void classifyIndexAndCollectStats( - Index index, - NodeStats nodeStats, - Map indexByTier, - RoutingNode node, - Map accumulators - ) { - // Look up which tier this index belongs to (its most preferred) - String indexTier = indexByTier.get(index.getName()); - if (indexTier != null) { - final TierStatsAccumulator accumulator = accumulators.computeIfAbsent(indexTier, k -> new TierStatsAccumulator()); - accumulator.indexNames.add(index.getName()); - aggregateDataTierShardStats(nodeStats, index, node, accumulator); - } - } - - /** - * Collect shard-level data tier stats from shard stats contained in the node stats response. - */ - private static void aggregateDataTierShardStats(NodeStats nodeStats, Index index, RoutingNode node, TierStatsAccumulator accumulator) { - // Shard based stats - final List allShardStats = nodeStats.getIndices().getShardStats(index); - if (allShardStats != null) { - for (IndexShardStats shardStat : allShardStats) { - accumulator.totalByteCount += shardStat.getTotal().getStore().totalDataSetSizeInBytes(); - accumulator.docCount += shardStat.getTotal().getDocs().getCount(); - - // Accumulate stats about started shards - ShardRouting shardRouting = node.getByShardId(shardStat.getShardId()); - if (shardRouting != null && shardRouting.state() == ShardRoutingState.STARTED) { - accumulator.totalShardCount += 1; - - // Accumulate stats about started primary shards - StoreStats primaryStoreStats = shardStat.getPrimary().getStore(); - if (primaryStoreStats != null) { - // if primaryStoreStats is null, it means there is no primary on the node in question - accumulator.primaryShardCount++; - long primarySize = primaryStoreStats.totalDataSetSizeInBytes(); - accumulator.primaryByteCount += primarySize; - accumulator.valueSketch.add(primarySize); - } + private static void aggregateDataTierIndexStats(NodeDataTiersUsage nodeDataTiersUsage, Map accumulators) { + for (Map.Entry entry : nodeDataTiersUsage.getUsageStatsByTier().entrySet()) { + String tier = entry.getKey(); + NodeDataTiersUsage.UsageStats usage = entry.getValue(); + if (DataTier.validTierName(tier)) { + TierStatsAccumulator accumulator = accumulators.computeIfAbsent(tier, k -> new TierStatsAccumulator()); + accumulator.indexNames.addAll(usage.getIndices()); + accumulator.docCount += usage.getDocCount(); + accumulator.totalByteCount += usage.getTotalSize(); + accumulator.totalShardCount += usage.getTotalShardCount(); + for (Long primaryShardSize : usage.getPrimaryShardSizes()) { + accumulator.primaryShardCount += 1; + accumulator.primaryByteCount += primaryShardSize; + accumulator.valueSketch.add(primaryShardSize); } } } } - private static DataTiersFeatureSetUsage.TierSpecificStats calculateFinalTierStats(TierStatsAccumulator accumulator) { + private static DataTiersFeatureSetUsage.TierSpecificStats aggregateFinalTierStats(TierStatsAccumulator accumulator) { long primaryShardSizeMedian = (long) accumulator.valueSketch.quantile(0.5); long primaryShardSizeMAD = computeMedianAbsoluteDeviation(accumulator.valueSketch); return new DataTiersFeatureSetUsage.TierSpecificStats( diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java index 40ad14245d152..d15b8d8fd9248 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java @@ -7,59 +7,34 @@ package org.elasticsearch.xpack.core.datatiers; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.ShardStats; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.cluster.routing.allocation.DataTier; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.shard.DocsStats; -import org.elasticsearch.index.shard.IndexLongFieldRange; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.ShardPath; -import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.test.ESTestCase; +import org.junit.Before; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.IntStream; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class DataTiersUsageTransportActionTests extends ESTestCase { + private long byteSize; + private long docCount; + + @Before + public void setup() { + byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB + docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million + } + public void testCalculateMAD() { assertThat(DataTiersUsageTransportAction.computeMedianAbsoluteDeviation(TDigestState.create(10)), equalTo(0L)); @@ -74,75 +49,17 @@ public void testCalculateMAD() { assertThat(DataTiersUsageTransportAction.computeMedianAbsoluteDeviation(sketch), equalTo(1L)); } - public void testTierIndices() { - IndexMetadata hotIndex1 = indexMetadata("hot-1", 1, 0, DataTier.DATA_HOT); - IndexMetadata hotIndex2 = indexMetadata("hot-2", 1, 0, DataTier.DATA_HOT); - IndexMetadata warmIndex1 = indexMetadata("warm-1", 1, 0, DataTier.DATA_WARM); - IndexMetadata coldIndex1 = indexMetadata("cold-1", 1, 0, DataTier.DATA_COLD); - IndexMetadata coldIndex2 = indexMetadata("cold-2", 1, 0, DataTier.DATA_COLD, DataTier.DATA_WARM); // Prefers cold over warm - IndexMetadata nonTiered = indexMetadata("non-tier", 1, 0); // No tier - - Map indices = new HashMap<>(); - indices.put("hot-1", hotIndex1); - indices.put("hot-2", hotIndex2); - indices.put("warm-1", warmIndex1); - indices.put("cold-1", coldIndex1); - indices.put("cold-2", coldIndex2); - indices.put("non-tier", nonTiered); - - Map tiers = DataTiersUsageTransportAction.tierIndices(indices); - assertThat(tiers.size(), equalTo(5)); - assertThat(tiers.get("hot-1"), equalTo(DataTier.DATA_HOT)); - assertThat(tiers.get("hot-2"), equalTo(DataTier.DATA_HOT)); - assertThat(tiers.get("warm-1"), equalTo(DataTier.DATA_WARM)); - assertThat(tiers.get("cold-1"), equalTo(DataTier.DATA_COLD)); - assertThat(tiers.get("cold-2"), equalTo(DataTier.DATA_COLD)); - assertThat(tiers.get("non-tier"), nullValue()); - } - public void testCalculateStatsNoTiers() { // Nodes: 0 Tiered Nodes, 1 Data Node - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); DiscoveryNode leader = newNode(0, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode1); - - discoBuilder.localNodeId(dataNode1.getId()); - - // Indices: 1 Regular index - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata index1 = indexMetadata("index_1", 3, 1); - metadataBuilder.put(index1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index1.getIndex()); - routeTestShardToNodes(index1, 0, indexRoutingTableBuilder, dataNode1); - routeTestShardToNodes(index1, 1, indexRoutingTableBuilder, dataNode1); - routeTestShardToNodes(index1, 2, indexRoutingTableBuilder, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); - // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage(dataNode1, Map.of()) + ); + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages ); // Verify - No results when no tiers present @@ -151,55 +68,24 @@ public void testCalculateStatsNoTiers() { public void testCalculateStatsTieredNodesOnly() { // Nodes: 1 Data, 1 Hot, 1 Warm, 1 Cold, 1 Frozen - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); DiscoveryNode leader = newNode(0, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode1); DiscoveryNode hotNode1 = newNode(2, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode1); DiscoveryNode warmNode1 = newNode(3, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode1); DiscoveryNode coldNode1 = newNode(4, DiscoveryNodeRole.DATA_COLD_NODE_ROLE); - discoBuilder.add(coldNode1); DiscoveryNode frozenNode1 = newNode(5, DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE); - discoBuilder.add(frozenNode1); - - discoBuilder.localNodeId(dataNode1.getId()); - - // Indices: 1 Regular index, not hosted on any tiers - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata index1 = indexMetadata("index_1", 3, 1); - metadataBuilder.put(index1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index1.getIndex()); - routeTestShardToNodes(index1, 0, indexRoutingTableBuilder, dataNode1); - routeTestShardToNodes(index1, 1, indexRoutingTableBuilder, dataNode1); - routeTestShardToNodes(index1, 2, indexRoutingTableBuilder, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); - // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage(dataNode1, Map.of()), + new NodeDataTiersUsage(hotNode1, Map.of()), + new NodeDataTiersUsage(warmNode1, Map.of()), + new NodeDataTiersUsage(coldNode1, Map.of()), + new NodeDataTiersUsage(frozenNode1, Map.of()) + ); + + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages ); // Verify - Results are present but they lack index numbers because none are tiered @@ -257,88 +143,61 @@ public void testCalculateStatsTieredNodesOnly() { public void testCalculateStatsTieredIndicesOnly() { // Nodes: 3 Data, 0 Tiered - Only hosting indices on generic data nodes int nodeId = 0; - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - DiscoveryNode dataNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode1); DiscoveryNode dataNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode2); DiscoveryNode dataNode3 = newNode(nodeId, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode3); - - discoBuilder.localNodeId(dataNode1.getId()); - - // Indices: 1 Hot index, 2 Warm indices, 3 Cold indices - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); - metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); - routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, dataNode1, dataNode2); - routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, dataNode2, dataNode3); - routeTestShardToNodes(hotIndex1, 2, indexRoutingTableBuilder, dataNode3, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); - routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, dataNode1, dataNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata warmIndex2 = indexMetadata("warm_index_2", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex2.getIndex()); - routeTestShardToNodes(warmIndex2, 0, indexRoutingTableBuilder, dataNode3, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata coldIndex1 = indexMetadata("cold_index_1", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex1.getIndex()); - routeTestShardToNodes(coldIndex1, 0, indexRoutingTableBuilder, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata coldIndex2 = indexMetadata("cold_index_2", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex2.getIndex()); - routeTestShardToNodes(coldIndex2, 0, indexRoutingTableBuilder, dataNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata coldIndex3 = indexMetadata("cold_index_3", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex3, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex3.getIndex()); - routeTestShardToNodes(coldIndex3, 0, indexRoutingTableBuilder, dataNode3); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); + // Indices: + // 1 Hot index: 3 primaries, 3 replicas one on each node + // 2 Warm indices, each index 1 primary one replica + // 3 Cold indices, each index 1 primary on a different node + String hotIndex = "hot_index_1"; + String warmIndex1 = "warm_index_1"; + String warmIndex2 = "warm_index_2"; + String coldIndex1 = "cold_index_1"; + String coldIndex2 = "cold_index_2"; + String coldIndex3 = "cold_index_3"; + + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage( + dataNode1, + Map.of( + DataTier.DATA_HOT, + createStats(Set.of(hotIndex), 1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(Set.of(warmIndex1, warmIndex2), 0, 2, docCount, byteSize), + DataTier.DATA_COLD, + createStats(Set.of(coldIndex1), 1, 1, docCount, byteSize) + ) + ), + new NodeDataTiersUsage( + dataNode2, + Map.of( + DataTier.DATA_HOT, + createStats(Set.of(hotIndex), 1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(Set.of(warmIndex1), 1, 1, docCount, byteSize), + DataTier.DATA_COLD, + createStats(Set.of(coldIndex2), 1, 1, docCount, byteSize) + ) + ), + new NodeDataTiersUsage( + dataNode3, + Map.of( + DataTier.DATA_HOT, + createStats(Set.of(hotIndex), 1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(Set.of(warmIndex2), 1, 1, docCount, byteSize), + DataTier.DATA_COLD, + createStats(Set.of(coldIndex3), 1, 1, docCount, byteSize) + ) + ) + ); // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages ); // Verify - Index stats exist for the tiers, but no tiered nodes are found @@ -384,100 +243,47 @@ public void testCalculateStatsTieredIndicesOnly() { public void testCalculateStatsReasonableCase() { // Nodes: 3 Hot, 5 Warm, 1 Cold int nodeId = 0; - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - DiscoveryNode hotNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode1); DiscoveryNode hotNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode2); DiscoveryNode hotNode3 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode3); DiscoveryNode warmNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode1); DiscoveryNode warmNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode2); DiscoveryNode warmNode3 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode3); DiscoveryNode warmNode4 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode4); DiscoveryNode warmNode5 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode5); DiscoveryNode coldNode1 = newNode(nodeId, DiscoveryNodeRole.DATA_COLD_NODE_ROLE); - discoBuilder.add(coldNode1); - - discoBuilder.localNodeId(hotNode1.getId()); - - // Indices: 1 Hot index, 2 Warm indices, 3 Cold indices - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); - metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); - routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, hotNode1, hotNode2); - routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, hotNode2, hotNode3); - routeTestShardToNodes(hotIndex1, 2, indexRoutingTableBuilder, hotNode3, hotNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); - routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, warmNode1, warmNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata warmIndex2 = indexMetadata("warm_index_2", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex2.getIndex()); - routeTestShardToNodes(warmIndex2, 0, indexRoutingTableBuilder, warmNode3, warmNode4); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata coldIndex1 = indexMetadata("cold_index_1", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex1.getIndex()); - routeTestShardToNodes(coldIndex1, 0, indexRoutingTableBuilder, coldNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata coldIndex2 = indexMetadata("cold_index_2", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex2.getIndex()); - routeTestShardToNodes(coldIndex2, 0, indexRoutingTableBuilder, coldNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata coldIndex3 = indexMetadata("cold_index_3", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex3, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex3.getIndex()); - routeTestShardToNodes(coldIndex3, 0, indexRoutingTableBuilder, coldNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); + // Indices: + // 1 Hot index: 3 primaries, 3 replicas one on each node + // 2 Warm indices: each index has 1 primary and 1 replica residing in 4 nodes + // 3 Cold indices: 1 primary each on the cold node + String hotIndex1 = "hot_index_1"; + String warmIndex1 = "warm_index_1"; + String warmIndex2 = "warm_index_2"; + String coldIndex1 = "cold_index_1"; + String coldIndex2 = "cold_index_2"; + String coldIndex3 = "cold_index_3"; + + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage(hotNode1, Map.of(DataTier.DATA_HOT, createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize))), + new NodeDataTiersUsage(hotNode2, Map.of(DataTier.DATA_HOT, createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize))), + new NodeDataTiersUsage(hotNode3, Map.of(DataTier.DATA_HOT, createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize))), + new NodeDataTiersUsage(warmNode1, Map.of(DataTier.DATA_WARM, createStats(Set.of(warmIndex1), 1, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode2, Map.of(DataTier.DATA_WARM, createStats(Set.of(warmIndex1), 0, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode3, Map.of(DataTier.DATA_WARM, createStats(Set.of(warmIndex2), 1, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode4, Map.of(DataTier.DATA_WARM, createStats(Set.of(warmIndex2), 0, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode5, Map.of()), + new NodeDataTiersUsage( + coldNode1, + Map.of(DataTier.DATA_COLD, createStats(Set.of(coldIndex1, coldIndex2, coldIndex3), 3, 3, docCount, byteSize)) + ) + + ); // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages ); // Verify - Node and Index stats are both collected @@ -523,66 +329,51 @@ public void testCalculateStatsReasonableCase() { public void testCalculateStatsMixedTiers() { // Nodes: 3 Hot+Warm - Nodes that are marked as part of multiple tiers int nodeId = 0; - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); DiscoveryNode mixedNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(mixedNode1); DiscoveryNode mixedNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(mixedNode2); DiscoveryNode mixedNode3 = newNode(nodeId, DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(mixedNode3); - discoBuilder.localNodeId(mixedNode1.getId()); + String hotIndex1 = "hot_index_1"; + String warmIndex1 = "warm_index_1"; + String warmIndex2 = "warm_index_2"; // Indices: 1 Hot index, 2 Warm indices - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); - metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); - routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, mixedNode1, mixedNode2); - routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, mixedNode3, mixedNode1); - routeTestShardToNodes(hotIndex1, 2, indexRoutingTableBuilder, mixedNode2, mixedNode3); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); - routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, mixedNode1, mixedNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata warmIndex2 = indexMetadata("warm_index_2", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex2.getIndex()); - routeTestShardToNodes(warmIndex2, 0, indexRoutingTableBuilder, mixedNode3, mixedNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage( + mixedNode1, + Map.of( + DataTier.DATA_HOT, + createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(Set.of(warmIndex1, warmIndex2), 1, 2, docCount, byteSize) + ) + ), + new NodeDataTiersUsage( + mixedNode2, + Map.of( + DataTier.DATA_HOT, + createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(Set.of(warmIndex1), 0, 1, docCount, byteSize) + ) + ), + new NodeDataTiersUsage( + mixedNode3, + Map.of( + DataTier.DATA_HOT, + createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(Set.of(warmIndex2), 1, 1, docCount, byteSize) + ) + ) + ); // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages ); // Verify - Index stats are separated by their preferred tier, instead of counted @@ -617,59 +408,40 @@ public void testCalculateStatsMixedTiers() { public void testCalculateStatsStuckInWrongTier() { // Nodes: 3 Hot, 0 Warm - Emulating indices stuck on non-preferred tiers int nodeId = 0; - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - DiscoveryNode hotNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode1); DiscoveryNode hotNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode2); DiscoveryNode hotNode3 = newNode(nodeId, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode3); - - discoBuilder.localNodeId(hotNode1.getId()); - - // Indices: 1 Hot index, 1 Warm index (Warm index is allocated to less preferred hot node because warm nodes are missing) - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); - metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); - routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, hotNode1, hotNode2); - routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, hotNode3, hotNode1); - routeTestShardToNodes(hotIndex1, 2, indexRoutingTableBuilder, hotNode2, hotNode3); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM, DataTier.DATA_HOT); - metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); - routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, hotNode1, hotNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); + + String hotIndex1 = "hot_index_1"; + String warmIndex1 = "warm_index_1"; + + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage( + hotNode1, + Map.of( + DataTier.DATA_HOT, + createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(Set.of(warmIndex1), 1, 1, docCount, byteSize) + ) + ), + new NodeDataTiersUsage( + hotNode2, + Map.of( + DataTier.DATA_HOT, + createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(Set.of(warmIndex1), 0, 1, docCount, byteSize) + ) + ), + new NodeDataTiersUsage(hotNode3, Map.of(DataTier.DATA_HOT, createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize))) + ); // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages ); // Verify - Warm indices are still calculated separately from Hot ones, despite Warm nodes missing @@ -704,83 +476,19 @@ private static DiscoveryNode newNode(int nodeId, DiscoveryNodeRole... roles) { return DiscoveryNodeUtils.builder("node_" + nodeId).roles(Set.of(roles)).build(); } - private static IndexMetadata indexMetadata(String indexName, int numberOfShards, int numberOfReplicas, String... dataTierPrefs) { - Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), numberOfShards, numberOfReplicas).put( - SETTING_CREATION_DATE, - System.currentTimeMillis() - ); - - if (dataTierPrefs.length > 1) { - StringBuilder tierBuilder = new StringBuilder(dataTierPrefs[0]); - for (int idx = 1; idx < dataTierPrefs.length; idx++) { - tierBuilder.append(',').append(dataTierPrefs[idx]); - } - settingsBuilder.put(DataTier.TIER_PREFERENCE, tierBuilder.toString()); - } else if (dataTierPrefs.length == 1) { - settingsBuilder.put(DataTier.TIER_PREFERENCE, dataTierPrefs[0]); - } - - return IndexMetadata.builder(indexName).settings(settingsBuilder.build()).timestampRange(IndexLongFieldRange.UNKNOWN).build(); - } - - private static void routeTestShardToNodes( - IndexMetadata index, - int shard, - IndexRoutingTable.Builder indexRoutingTableBuilder, - DiscoveryNode... nodes + private NodeDataTiersUsage.UsageStats createStats( + Set indices, + int primaryShardCount, + int totalNumberOfShards, + long docCount, + long byteSize ) { - ShardId shardId = new ShardId(index.getIndex(), shard); - IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); - boolean primary = true; - for (DiscoveryNode node : nodes) { - indexShardRoutingBuilder.addShard( - TestShardRouting.newShardRouting(shardId, node.getId(), null, primary, ShardRoutingState.STARTED) - ); - primary = false; - } - indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder); - } - - private List buildNodeStats(ClusterState clusterState, long bytesPerShard, long docsPerShard) { - DiscoveryNodes nodes = clusterState.getNodes(); - RoutingNodes routingNodes = clusterState.getRoutingNodes(); - List nodeStatsList = new ArrayList<>(); - for (DiscoveryNode node : nodes) { - RoutingNode routingNode = routingNodes.node(node.getId()); - if (routingNode == null) { - continue; - } - Map> indexStats = new HashMap<>(); - for (ShardRouting shardRouting : routingNode) { - ShardId shardId = shardRouting.shardId(); - ShardStats shardStat = shardStat(bytesPerShard, docsPerShard, shardRouting); - IndexShardStats shardStats = new IndexShardStats(shardId, new ShardStats[] { shardStat }); - indexStats.computeIfAbsent(shardId.getIndex(), k -> new ArrayList<>()).add(shardStats); - } - NodeIndicesStats nodeIndexStats = new NodeIndicesStats(new CommonStats(), Collections.emptyMap(), indexStats, true); - nodeStatsList.add(mockNodeStats(node, nodeIndexStats)); - } - return nodeStatsList; - } - - private static ShardStats shardStat(long byteCount, long docCount, ShardRouting routing) { - StoreStats storeStats = new StoreStats(randomNonNegativeLong(), byteCount, 0L); - DocsStats docsStats = new DocsStats(docCount, 0L, byteCount); - - CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL); - commonStats.getStore().add(storeStats); - commonStats.getDocs().add(docsStats); - - Path fakePath = PathUtils.get("test/dir/" + routing.shardId().getIndex().getUUID() + "/" + routing.shardId().id()); - ShardPath fakeShardPath = new ShardPath(false, fakePath, fakePath, routing.shardId()); - - return new ShardStats(routing, fakeShardPath, commonStats, null, null, null, false, 0); - } - - private static NodeStats mockNodeStats(DiscoveryNode node, NodeIndicesStats indexStats) { - NodeStats stats = mock(NodeStats.class); - when(stats.getNode()).thenReturn(node); - when(stats.getIndices()).thenReturn(indexStats); - return stats; + return new NodeDataTiersUsage.UsageStats( + indices, + primaryShardCount > 0 ? IntStream.range(0, primaryShardCount).mapToObj(i -> byteSize).toList() : List.of(), + totalNumberOfShards, + totalNumberOfShards * docCount, + totalNumberOfShards * byteSize + ); } } From 6fad632219e354eb87ff65b21d51c0873ae62814 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 1 Nov 2023 13:40:14 +0200 Subject: [PATCH 04/12] Bug fix: do not overwrite the CommonStatsFlags.NONE --- .../core/datatiers/NodesDataTiersUsageTransportAction.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java index 2a3787a6e5acb..904cf83facf8e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java @@ -57,6 +57,9 @@ public class NodesDataTiersUsageTransportAction extends TransportNodesAction< NodeDataTiersUsage> { public static final ActionType TYPE = ActionType.localOnly("cluster:monitor/nodes/data_tier_usage"); + private static final CommonStatsFlags STATS_FLAGS = new CommonStatsFlags().clear() + .set(CommonStatsFlags.Flag.Docs, true) + .set(CommonStatsFlags.Flag.Store, true); private final IndicesService indicesService; @@ -99,8 +102,7 @@ protected NodeDataTiersUsage nodeOperation(NodeRequest nodeRequest, Task task) { assert task instanceof CancellableTask; DiscoveryNode localNode = clusterService.localNode(); - CommonStatsFlags flags = CommonStatsFlags.NONE.set(CommonStatsFlags.Flag.Docs, true).set(CommonStatsFlags.Flag.Store, true); - NodeIndicesStats nodeIndicesStats = indicesService.stats(flags, true); + NodeIndicesStats nodeIndicesStats = indicesService.stats(STATS_FLAGS, true); ClusterState state = clusterService.state(); RoutingNode routingNode = state.getRoutingNodes().node(localNode.getId()); Map usageStatsByTier = aggregateStats(routingNode, state.metadata(), nodeIndicesStats); From 01334b03ee14ee7c4c0d4cf075a138dc58a3894f Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 1 Nov 2023 13:41:25 +0200 Subject: [PATCH 05/12] Update docs/changelog/101599.yaml --- docs/changelog/101599.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/101599.yaml diff --git a/docs/changelog/101599.yaml b/docs/changelog/101599.yaml new file mode 100644 index 0000000000000..9fa1b49b09f35 --- /dev/null +++ b/docs/changelog/101599.yaml @@ -0,0 +1,5 @@ +pr: 101599 +summary: Move the calculation of data tier usage stats to individual nodes +area: ILM+SLM +type: bug +issues: [] From f94153047563194eccb8d708b3273f6d0a4caeec Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 1 Nov 2023 13:42:55 +0200 Subject: [PATCH 06/12] Update 101599.yaml --- docs/changelog/101599.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog/101599.yaml b/docs/changelog/101599.yaml index 9fa1b49b09f35..968d40179342f 100644 --- a/docs/changelog/101599.yaml +++ b/docs/changelog/101599.yaml @@ -2,4 +2,5 @@ pr: 101599 summary: Move the calculation of data tier usage stats to individual nodes area: ILM+SLM type: bug -issues: [] +issues: + - 100230 From 5089cb431fe3e77e17fcf84d03a820d6d0a201d8 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 1 Nov 2023 13:43:00 +0200 Subject: [PATCH 07/12] Update docs/changelog/101599.yaml --- docs/changelog/101599.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog/101599.yaml b/docs/changelog/101599.yaml index 968d40179342f..9fa1b49b09f35 100644 --- a/docs/changelog/101599.yaml +++ b/docs/changelog/101599.yaml @@ -2,5 +2,4 @@ pr: 101599 summary: Move the calculation of data tier usage stats to individual nodes area: ILM+SLM type: bug -issues: - - 100230 +issues: [] From c8164bfe37676a55cfd93a1a7293b213397a8c4e Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 1 Nov 2023 13:45:14 +0200 Subject: [PATCH 08/12] Update 101599.yaml --- docs/changelog/101599.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog/101599.yaml b/docs/changelog/101599.yaml index 9fa1b49b09f35..4fb1c972eb083 100644 --- a/docs/changelog/101599.yaml +++ b/docs/changelog/101599.yaml @@ -2,4 +2,5 @@ pr: 101599 summary: Move the calculation of data tier usage stats to individual nodes area: ILM+SLM type: bug -issues: [] +issues: + - 100230 From 15d54cba7756d39b8ddecc2869e25d7895d2c294 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 1 Nov 2023 14:24:44 +0200 Subject: [PATCH 09/12] Update docs/changelog/101599.yaml --- docs/changelog/101599.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog/101599.yaml b/docs/changelog/101599.yaml index 4fb1c972eb083..9fa1b49b09f35 100644 --- a/docs/changelog/101599.yaml +++ b/docs/changelog/101599.yaml @@ -2,5 +2,4 @@ pr: 101599 summary: Move the calculation of data tier usage stats to individual nodes area: ILM+SLM type: bug -issues: - - 100230 +issues: [] From ebfb9a00aebeb8a5f2f9813f6564c40636bfdb90 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 8 Nov 2023 10:09:57 +0200 Subject: [PATCH 10/12] Use IndexMetadata#tierPreference --- .../NodesDataTiersUsageTransportAction.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java index 904cf83facf8e..6429067d71d63 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java @@ -23,12 +23,10 @@ import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.Nullable; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.NodeIndicesStats; @@ -124,7 +122,7 @@ static Map aggregateStats( .collect(Collectors.toSet()); for (String indexName : localIndices) { IndexMetadata indexMetadata = metadata.index(indexName); - String tier = findPreferredTier(indexMetadata); + String tier = indexMetadata.getTierPreference().isEmpty() ? null : indexMetadata.getTierPreference().get(0); if (tier != null) { NodeDataTiersUsage.UsageStats usageStats = usageStatsByTier.computeIfAbsent( tier, @@ -154,18 +152,6 @@ static Map aggregateStats( return usageStatsByTier; } - @Nullable - static String findPreferredTier(IndexMetadata indexMetadata) { - List tierPref = DataTier.parseTierList(indexMetadata.getSettings().get(DataTier.TIER_PREFERENCE)) - .stream() - .filter(DataTier::validTierName) - .toList(); - if (tierPref.isEmpty() == false) { - return tierPref.get(0); - } - return null; - } - public static class NodesRequest extends BaseNodesRequest { public NodesRequest() { From da072edb9d3180342ac2c19ab7f5962cb5453d33 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 8 Nov 2023 15:28:57 +0200 Subject: [PATCH 11/12] Move index grouping to coordinating node --- docs/changelog/101599.yaml | 3 +- .../DataTiersUsageTransportAction.java | 41 +++- .../core/datatiers/NodeDataTiersUsage.java | 24 +-- .../NodesDataTiersUsageTransportAction.java | 1 - .../core/datatiers/DataTierUsageFixtures.java | 114 +++++++++++ .../DataTiersUsageTransportActionTests.java | 185 +++++++++++------- ...desDataTiersUsageTransportActionTests.java | 118 +---------- 7 files changed, 273 insertions(+), 213 deletions(-) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java diff --git a/docs/changelog/101599.yaml b/docs/changelog/101599.yaml index 9fa1b49b09f35..4fb1c972eb083 100644 --- a/docs/changelog/101599.yaml +++ b/docs/changelog/101599.yaml @@ -2,4 +2,5 @@ pr: 101599 summary: Move the calculation of data tier usage stats to individual nodes area: ILM+SLM type: bug -issues: [] +issues: + - 100230 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java index 97b1b44312ab1..b437108919a89 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; @@ -30,7 +31,10 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; public class DataTiersUsageTransportAction extends XPackUsageFeatureTransportAction { @@ -70,11 +74,35 @@ protected void masterOperation( new NodesDataTiersUsageTransportAction.NodesRequest(), listener.delegateFailureAndWrap((delegate, response) -> { // Generate tier specific stats for the nodes and indices - delegate.onResponse(new XPackUsageFeatureResponse(new DataTiersFeatureSetUsage(aggregateStats(response.getNodes())))); + delegate.onResponse( + new XPackUsageFeatureResponse( + new DataTiersFeatureSetUsage( + aggregateStats(response.getNodes(), getIndicesGroupedByTier(state, response.getNodes())) + ) + ) + ); }) ); } + // Visible for testing + static Map> getIndicesGroupedByTier(ClusterState state, List nodes) { + Set indices = nodes.stream() + .map(nodeResponse -> state.getRoutingNodes().node(nodeResponse.getNode().getId())) + .filter(Objects::nonNull) + .flatMap(node -> StreamSupport.stream(node.spliterator(), false)) + .map(ShardRouting::getIndexName) + .collect(Collectors.toSet()); + Map> indicesByTierPreference = new HashMap<>(); + for (String indexName : indices) { + List tierPreference = state.metadata().index(indexName).getTierPreference(); + if (tierPreference.isEmpty() == false) { + indicesByTierPreference.computeIfAbsent(tierPreference.get(0), ignored -> new HashSet<>()).add(indexName); + } + } + return indicesByTierPreference; + } + /** * Accumulator to hold intermediate data tier stats before final calculation. */ @@ -90,10 +118,16 @@ private static class TierStatsAccumulator { } // Visible for testing - static Map aggregateStats(List nodeDataTiersUsages) { + static Map aggregateStats( + List nodeDataTiersUsages, + Map> tierPreference + ) { Map statsAccumulators = new HashMap<>(); + for (String tier : tierPreference.keySet()) { + statsAccumulators.put(tier, new TierStatsAccumulator()); + statsAccumulators.get(tier).indexNames.addAll(tierPreference.get(tier)); + } for (NodeDataTiersUsage nodeDataTiersUsage : nodeDataTiersUsages) { - // This ensures we only count the nodes that responded aggregateDataTierNodeCounts(nodeDataTiersUsage, statsAccumulators); aggregateDataTierIndexStats(nodeDataTiersUsage, statsAccumulators); } @@ -125,7 +159,6 @@ private static void aggregateDataTierIndexStats(NodeDataTiersUsage nodeDataTiers NodeDataTiersUsage.UsageStats usage = entry.getValue(); if (DataTier.validTierName(tier)) { TierStatsAccumulator accumulator = accumulators.computeIfAbsent(tier, k -> new TierStatsAccumulator()); - accumulator.indexNames.addAll(usage.getIndices()); accumulator.docCount += usage.getDocCount(); accumulator.totalByteCount += usage.getTotalSize(); accumulator.totalShardCount += usage.getTotalShardCount(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java index d99bf19bcf586..c1903a2910629 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java @@ -15,10 +15,8 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; /** * Data tier usage statistics on a specific node. The statistics groups the indices, shard sizes, shard counts based @@ -29,22 +27,19 @@ public class NodeDataTiersUsage extends BaseNodeResponse { private final Map usageStatsByTier; public static class UsageStats implements Writeable { - private final Set indices; private final List primaryShardSizes; private int totalShardCount; private long docCount; private long totalSize; public UsageStats() { - this.indices = new HashSet<>(); this.primaryShardSizes = new ArrayList<>(); this.totalShardCount = 0; this.docCount = 0; this.totalSize = 0; } - public UsageStats(Set indices, List primaryShardSizes, int totalShardCount, long docCount, long totalSize) { - this.indices = indices; + public UsageStats(List primaryShardSizes, int totalShardCount, long docCount, long totalSize) { this.primaryShardSizes = primaryShardSizes; this.totalShardCount = totalShardCount; this.docCount = docCount; @@ -52,28 +47,17 @@ public UsageStats(Set indices, List primaryShardSizes, int totalSh } static UsageStats read(StreamInput in) throws IOException { - return new UsageStats( - in.readCollectionAsSet(StreamInput::readString), - in.readCollectionAsList(StreamInput::readVLong), - in.readVInt(), - in.readVLong(), - in.readVLong() - ); + return new UsageStats(in.readCollectionAsList(StreamInput::readVLong), in.readVInt(), in.readVLong(), in.readVLong()); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeCollection(indices, StreamOutput::writeString); out.writeCollection(primaryShardSizes, StreamOutput::writeVLong); out.writeVInt(totalShardCount); out.writeVLong(docCount); out.writeVLong(totalSize); } - public void addIndex(String indexName) { - indices.add(indexName); - } - public void addPrimaryShardSize(long primaryShardSize) { primaryShardSizes.add(primaryShardSize); } @@ -90,10 +74,6 @@ public void incrementTotalShardCount(int totalShardCount) { this.totalShardCount += totalShardCount; } - public Set getIndices() { - return indices; - } - public List getPrimaryShardSizes() { return primaryShardSizes; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java index 6429067d71d63..e4d853f4e6050 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java @@ -128,7 +128,6 @@ static Map aggregateStats( tier, ignored -> new NodeDataTiersUsage.UsageStats() ); - usageStats.addIndex(indexName); List allShardStats = nodeIndicesStats.getShardStats(indexMetadata.getIndex()); if (allShardStats != null) { for (IndexShardStats indexShardStats : allShardStats) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java new file mode 100644 index 0000000000000..63cc6e4d7914e --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.IndexLongFieldRange; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.test.ESTestCase; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; + +class DataTierUsageFixtures extends ESTestCase { + + private static final CommonStats COMMON_STATS = new CommonStats( + CommonStatsFlags.NONE.set(CommonStatsFlags.Flag.Docs, true).set(CommonStatsFlags.Flag.Store, true) + ); + + static DiscoveryNode newNode(int nodeId, DiscoveryNodeRole... roles) { + return DiscoveryNodeUtils.builder("node_" + nodeId).roles(Set.of(roles)).build(); + } + + static void routeTestShardToNodes( + IndexMetadata index, + int shard, + IndexRoutingTable.Builder indexRoutingTableBuilder, + DiscoveryNode... nodes + ) { + ShardId shardId = new ShardId(index.getIndex(), shard); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); + boolean primary = true; + for (DiscoveryNode node : nodes) { + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(shardId, node.getId(), null, primary, ShardRoutingState.STARTED) + ); + primary = false; + } + indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder); + } + + static NodeIndicesStats buildNodeIndicesStats(RoutingNode routingNode, long bytesPerShard, long docsPerShard) { + Map> indexStats = new HashMap<>(); + for (ShardRouting shardRouting : routingNode) { + ShardId shardId = shardRouting.shardId(); + ShardStats shardStat = shardStat(bytesPerShard, docsPerShard, shardRouting); + IndexShardStats shardStats = new IndexShardStats(shardId, new ShardStats[] { shardStat }); + indexStats.computeIfAbsent(shardId.getIndex(), k -> new ArrayList<>()).add(shardStats); + } + return new NodeIndicesStats(COMMON_STATS, Map.of(), indexStats, true); + } + + private static ShardStats shardStat(long byteCount, long docCount, ShardRouting routing) { + StoreStats storeStats = new StoreStats(randomNonNegativeLong(), byteCount, 0L); + DocsStats docsStats = new DocsStats(docCount, 0L, byteCount); + Path fakePath = PathUtils.get("test/dir/" + routing.shardId().getIndex().getUUID() + "/" + routing.shardId().id()); + ShardPath fakeShardPath = new ShardPath(false, fakePath, fakePath, routing.shardId()); + CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL); + commonStats.getStore().add(storeStats); + commonStats.getDocs().add(docsStats); + return new ShardStats(routing, fakeShardPath, commonStats, null, null, null, false, 0); + } + + static IndexMetadata indexMetadata(String indexName, int numberOfShards, int numberOfReplicas, String... dataTierPrefs) { + Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), numberOfShards, numberOfReplicas).put( + SETTING_CREATION_DATE, + System.currentTimeMillis() + ); + + if (dataTierPrefs.length > 1) { + StringBuilder tierBuilder = new StringBuilder(dataTierPrefs[0]); + for (int idx = 1; idx < dataTierPrefs.length; idx++) { + tierBuilder.append(',').append(dataTierPrefs[idx]); + } + settingsBuilder.put(DataTier.TIER_PREFERENCE, tierBuilder.toString()); + } else if (dataTierPrefs.length == 1) { + settingsBuilder.put(DataTier.TIER_PREFERENCE, dataTierPrefs[0]); + } + + return IndexMetadata.builder(indexName).settings(settingsBuilder.build()).timestampRange(IndexLongFieldRange.UNKNOWN).build(); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java index d15b8d8fd9248..bb8dce7db0e23 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java @@ -7,9 +7,15 @@ package org.elasticsearch.xpack.core.datatiers; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.test.ESTestCase; @@ -20,6 +26,9 @@ import java.util.Set; import java.util.stream.IntStream; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.indexMetadata; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.newNode; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.routeTestShardToNodes; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -35,6 +44,51 @@ public void setup() { docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million } + public void testTierIndices() { + DiscoveryNode dataNode = newNode(0, DiscoveryNodeRole.DATA_ROLE); + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + discoBuilder.add(dataNode); + + IndexMetadata hotIndex1 = indexMetadata("hot-1", 1, 0, DataTier.DATA_HOT); + IndexMetadata hotIndex2 = indexMetadata("hot-2", 1, 0, DataTier.DATA_HOT); + IndexMetadata warmIndex1 = indexMetadata("warm-1", 1, 0, DataTier.DATA_WARM); + IndexMetadata coldIndex1 = indexMetadata("cold-1", 1, 0, DataTier.DATA_COLD); + IndexMetadata coldIndex2 = indexMetadata("cold-2", 1, 0, DataTier.DATA_COLD, DataTier.DATA_WARM); // Prefers cold over warm + IndexMetadata nonTiered = indexMetadata("non-tier", 1, 0); // No tier + IndexMetadata hotIndex3 = indexMetadata("hot-3", 1, 0, DataTier.DATA_HOT); + + Metadata.Builder metadataBuilder = Metadata.builder() + .put(hotIndex1, false) + .put(hotIndex2, false) + .put(warmIndex1, false) + .put(coldIndex1, false) + .put(coldIndex2, false) + .put(nonTiered, false) + .put(hotIndex3, false) + .generateClusterUuidIfNeeded(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + routingTableBuilder.add(getIndexRoutingTable(hotIndex1, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(hotIndex2, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(hotIndex2, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(warmIndex1, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(coldIndex1, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(coldIndex2, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(nonTiered, dataNode)); + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .nodes(discoBuilder) + .metadata(metadataBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + Map> result = DataTiersUsageTransportAction.getIndicesGroupedByTier( + clusterState, + List.of(new NodeDataTiersUsage(dataNode, Map.of(DataTier.DATA_WARM, createStats(5, 5, 0, 10)))) + ); + assertThat(result.keySet(), equalTo(Set.of(DataTier.DATA_HOT, DataTier.DATA_WARM, DataTier.DATA_COLD))); + assertThat(result.get(DataTier.DATA_HOT), equalTo(Set.of(hotIndex1.getIndex().getName(), hotIndex2.getIndex().getName()))); + assertThat(result.get(DataTier.DATA_WARM), equalTo(Set.of(warmIndex1.getIndex().getName()))); + assertThat(result.get(DataTier.DATA_COLD), equalTo(Set.of(coldIndex1.getIndex().getName(), coldIndex2.getIndex().getName()))); + } + public void testCalculateMAD() { assertThat(DataTiersUsageTransportAction.computeMedianAbsoluteDeviation(TDigestState.create(10)), equalTo(0L)); @@ -50,7 +104,7 @@ public void testCalculateMAD() { } public void testCalculateStatsNoTiers() { - // Nodes: 0 Tiered Nodes, 1 Data Node + // Nodes: 0 Tiered Nodes, 1 Data Node, no indices on tiered nodes DiscoveryNode leader = newNode(0, DiscoveryNodeRole.MASTER_ROLE); DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_ROLE); @@ -59,7 +113,8 @@ public void testCalculateStatsNoTiers() { new NodeDataTiersUsage(dataNode1, Map.of()) ); Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( - nodeDataTiersUsages + nodeDataTiersUsages, + Map.of() ); // Verify - No results when no tiers present @@ -85,10 +140,11 @@ public void testCalculateStatsTieredNodesOnly() { ); Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( - nodeDataTiersUsages + nodeDataTiersUsages, + Map.of() ); - // Verify - Results are present but they lack index numbers because none are tiered + // Verify - Results are present, but they lack index numbers because none are tiered assertThat(tierSpecificStats.size(), is(4)); DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); @@ -165,39 +221,47 @@ public void testCalculateStatsTieredIndicesOnly() { dataNode1, Map.of( DataTier.DATA_HOT, - createStats(Set.of(hotIndex), 1, 2, docCount, byteSize), + createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, - createStats(Set.of(warmIndex1, warmIndex2), 0, 2, docCount, byteSize), + createStats(0, 2, docCount, byteSize), DataTier.DATA_COLD, - createStats(Set.of(coldIndex1), 1, 1, docCount, byteSize) + createStats(1, 1, docCount, byteSize) ) ), new NodeDataTiersUsage( dataNode2, Map.of( DataTier.DATA_HOT, - createStats(Set.of(hotIndex), 1, 2, docCount, byteSize), + createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, - createStats(Set.of(warmIndex1), 1, 1, docCount, byteSize), + createStats(1, 1, docCount, byteSize), DataTier.DATA_COLD, - createStats(Set.of(coldIndex2), 1, 1, docCount, byteSize) + createStats(1, 1, docCount, byteSize) ) ), new NodeDataTiersUsage( dataNode3, Map.of( DataTier.DATA_HOT, - createStats(Set.of(hotIndex), 1, 2, docCount, byteSize), + createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, - createStats(Set.of(warmIndex2), 1, 1, docCount, byteSize), + createStats(1, 1, docCount, byteSize), DataTier.DATA_COLD, - createStats(Set.of(coldIndex3), 1, 1, docCount, byteSize) + createStats(1, 1, docCount, byteSize) ) ) ); // Calculate usage Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( - nodeDataTiersUsages + nodeDataTiersUsages, + Map.of( + DataTier.DATA_HOT, + Set.of(hotIndex), + DataTier.DATA_WARM, + Set.of(warmIndex1, warmIndex2), + DataTier.DATA_COLD, + Set.of(coldIndex1, coldIndex2, coldIndex3) + ) ); // Verify - Index stats exist for the tiers, but no tiered nodes are found @@ -267,23 +331,28 @@ public void testCalculateStatsReasonableCase() { List nodeDataTiersUsages = List.of( new NodeDataTiersUsage(leader, Map.of()), - new NodeDataTiersUsage(hotNode1, Map.of(DataTier.DATA_HOT, createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize))), - new NodeDataTiersUsage(hotNode2, Map.of(DataTier.DATA_HOT, createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize))), - new NodeDataTiersUsage(hotNode3, Map.of(DataTier.DATA_HOT, createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize))), - new NodeDataTiersUsage(warmNode1, Map.of(DataTier.DATA_WARM, createStats(Set.of(warmIndex1), 1, 1, docCount, byteSize))), - new NodeDataTiersUsage(warmNode2, Map.of(DataTier.DATA_WARM, createStats(Set.of(warmIndex1), 0, 1, docCount, byteSize))), - new NodeDataTiersUsage(warmNode3, Map.of(DataTier.DATA_WARM, createStats(Set.of(warmIndex2), 1, 1, docCount, byteSize))), - new NodeDataTiersUsage(warmNode4, Map.of(DataTier.DATA_WARM, createStats(Set.of(warmIndex2), 0, 1, docCount, byteSize))), + new NodeDataTiersUsage(hotNode1, Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize))), + new NodeDataTiersUsage(hotNode2, Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize))), + new NodeDataTiersUsage(hotNode3, Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize))), + new NodeDataTiersUsage(warmNode1, Map.of(DataTier.DATA_WARM, createStats(1, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode2, Map.of(DataTier.DATA_WARM, createStats(0, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode3, Map.of(DataTier.DATA_WARM, createStats(1, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode4, Map.of(DataTier.DATA_WARM, createStats(0, 1, docCount, byteSize))), new NodeDataTiersUsage(warmNode5, Map.of()), - new NodeDataTiersUsage( - coldNode1, - Map.of(DataTier.DATA_COLD, createStats(Set.of(coldIndex1, coldIndex2, coldIndex3), 3, 3, docCount, byteSize)) - ) + new NodeDataTiersUsage(coldNode1, Map.of(DataTier.DATA_COLD, createStats(3, 3, docCount, byteSize))) ); // Calculate usage Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( - nodeDataTiersUsages + nodeDataTiersUsages, + Map.of( + DataTier.DATA_HOT, + Set.of(hotIndex1), + DataTier.DATA_WARM, + Set.of(warmIndex1, warmIndex2), + DataTier.DATA_COLD, + Set.of(coldIndex1, coldIndex2, coldIndex3) + ) ); // Verify - Node and Index stats are both collected @@ -344,36 +413,22 @@ public void testCalculateStatsMixedTiers() { new NodeDataTiersUsage(leader, Map.of()), new NodeDataTiersUsage( mixedNode1, - Map.of( - DataTier.DATA_HOT, - createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), - DataTier.DATA_WARM, - createStats(Set.of(warmIndex1, warmIndex2), 1, 2, docCount, byteSize) - ) + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(1, 2, docCount, byteSize)) ), new NodeDataTiersUsage( mixedNode2, - Map.of( - DataTier.DATA_HOT, - createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), - DataTier.DATA_WARM, - createStats(Set.of(warmIndex1), 0, 1, docCount, byteSize) - ) + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(0, 1, docCount, byteSize)) ), new NodeDataTiersUsage( mixedNode3, - Map.of( - DataTier.DATA_HOT, - createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), - DataTier.DATA_WARM, - createStats(Set.of(warmIndex2), 1, 1, docCount, byteSize) - ) + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(1, 1, docCount, byteSize)) ) ); // Calculate usage Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( - nodeDataTiersUsages + nodeDataTiersUsages, + Map.of(DataTier.DATA_HOT, Set.of(hotIndex1), DataTier.DATA_WARM, Set.of(warmIndex1, warmIndex2)) ); // Verify - Index stats are separated by their preferred tier, instead of counted @@ -420,28 +475,19 @@ public void testCalculateStatsStuckInWrongTier() { new NodeDataTiersUsage(leader, Map.of()), new NodeDataTiersUsage( hotNode1, - Map.of( - DataTier.DATA_HOT, - createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), - DataTier.DATA_WARM, - createStats(Set.of(warmIndex1), 1, 1, docCount, byteSize) - ) + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(1, 1, docCount, byteSize)) ), new NodeDataTiersUsage( hotNode2, - Map.of( - DataTier.DATA_HOT, - createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize), - DataTier.DATA_WARM, - createStats(Set.of(warmIndex1), 0, 1, docCount, byteSize) - ) + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(0, 1, docCount, byteSize)) ), - new NodeDataTiersUsage(hotNode3, Map.of(DataTier.DATA_HOT, createStats(Set.of(hotIndex1), 1, 2, docCount, byteSize))) + new NodeDataTiersUsage(hotNode3, Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize))) ); // Calculate usage Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( - nodeDataTiersUsages + nodeDataTiersUsages, + Map.of(DataTier.DATA_HOT, Set.of(hotIndex1), DataTier.DATA_WARM, Set.of(warmIndex1)) ); // Verify - Warm indices are still calculated separately from Hot ones, despite Warm nodes missing @@ -472,23 +518,18 @@ public void testCalculateStatsStuckInWrongTier() { assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size } - private static DiscoveryNode newNode(int nodeId, DiscoveryNodeRole... roles) { - return DiscoveryNodeUtils.builder("node_" + nodeId).roles(Set.of(roles)).build(); - } - - private NodeDataTiersUsage.UsageStats createStats( - Set indices, - int primaryShardCount, - int totalNumberOfShards, - long docCount, - long byteSize - ) { + private NodeDataTiersUsage.UsageStats createStats(int primaryShardCount, int totalNumberOfShards, long docCount, long byteSize) { return new NodeDataTiersUsage.UsageStats( - indices, primaryShardCount > 0 ? IntStream.range(0, primaryShardCount).mapToObj(i -> byteSize).toList() : List.of(), totalNumberOfShards, totalNumberOfShards * docCount, totalNumberOfShards * byteSize ); } + + private IndexRoutingTable.Builder getIndexRoutingTable(IndexMetadata indexMetadata, DiscoveryNode node) { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetadata.getIndex()); + routeTestShardToNodes(indexMetadata, 0, indexRoutingTableBuilder, node); + return indexRoutingTableBuilder; + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java index dff234c3c832d..fb4291530d037 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java @@ -7,57 +7,33 @@ package org.elasticsearch.xpack.core.datatiers; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.cluster.routing.allocation.DataTier; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.shard.DocsStats; -import org.elasticsearch.index.shard.IndexLongFieldRange; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.ShardPath; -import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.buildNodeIndicesStats; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.indexMetadata; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.newNode; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.routeTestShardToNodes; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; public class NodesDataTiersUsageTransportActionTests extends ESTestCase { - private static final CommonStats COMMON_STATS = new CommonStats( - CommonStatsFlags.NONE.set(CommonStatsFlags.Flag.Docs, true).set(CommonStatsFlags.Flag.Store, true) - ); private long byteSize; private long docCount; @@ -67,24 +43,6 @@ public void setup() { docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million } - public void testTierIndices() { - IndexMetadata hotIndex1 = indexMetadata("hot-1", 1, 0, DataTier.DATA_HOT); - IndexMetadata hotIndex2 = indexMetadata("hot-2", 1, 0, DataTier.DATA_HOT); - IndexMetadata warmIndex1 = indexMetadata("warm-1", 1, 0, DataTier.DATA_WARM); - IndexMetadata coldIndex1 = indexMetadata("cold-1", 1, 0, DataTier.DATA_COLD); - IndexMetadata coldIndex2 = indexMetadata("cold-2", 1, 0, DataTier.DATA_COLD, DataTier.DATA_WARM); // Prefers cold over warm - IndexMetadata nonTiered = indexMetadata("non-tier", 1, 0); // No tier - IndexMetadata invalidTiered = indexMetadata("non-tier", 1, 0, "invalid-tier"); // No tier - - assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(hotIndex1), equalTo(DataTier.DATA_HOT)); - assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(hotIndex2), equalTo(DataTier.DATA_HOT)); - assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(warmIndex1), equalTo(DataTier.DATA_WARM)); - assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(coldIndex1), equalTo(DataTier.DATA_COLD)); - assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(coldIndex2), equalTo(DataTier.DATA_COLD)); - assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(nonTiered), nullValue()); - assertThat(NodesDataTiersUsageTransportAction.findPreferredTier(invalidTiered), nullValue()); - } - public void testCalculateStatsNoTiers() { // Nodes: 0 Tiered Nodes, 1 Data Node DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); @@ -169,7 +127,7 @@ public void testCalculateStatsTieredIndicesOnly() { DiscoveryNode dataNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); discoBuilder.add(dataNode1); - DiscoveryNode dataNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); + DiscoveryNode dataNode2 = newNode(nodeId, DiscoveryNodeRole.DATA_ROLE); discoBuilder.add(dataNode2); discoBuilder.localNodeId(dataNode1.getId()); @@ -234,7 +192,6 @@ public void testCalculateStatsTieredIndicesOnly() { NodeDataTiersUsage.UsageStats hotStats = usageStats.get(DataTier.DATA_HOT); assertThat(hotStats, is(notNullValue())); - assertThat(hotStats.getIndices(), equalTo(Set.of(hotIndex1.getIndex().getName()))); assertThat(hotStats.getPrimaryShardSizes(), equalTo(List.of(byteSize))); assertThat(hotStats.getTotalShardCount(), is(2)); assertThat(hotStats.getDocCount(), is(hotStats.getTotalShardCount() * docCount)); @@ -242,7 +199,6 @@ public void testCalculateStatsTieredIndicesOnly() { NodeDataTiersUsage.UsageStats warmStats = usageStats.get(DataTier.DATA_WARM); assertThat(warmStats, is(notNullValue())); - assertThat(warmStats.getIndices(), equalTo(Set.of(warmIndex1.getIndex().getName(), warmIndex2.getIndex().getName()))); assertThat(warmStats.getPrimaryShardSizes(), equalTo(List.of(byteSize))); assertThat(warmStats.getTotalShardCount(), is(2)); assertThat(warmStats.getDocCount(), is(warmStats.getTotalShardCount() * docCount)); @@ -250,73 +206,9 @@ public void testCalculateStatsTieredIndicesOnly() { NodeDataTiersUsage.UsageStats coldStats = usageStats.get(DataTier.DATA_COLD); assertThat(coldStats, is(notNullValue())); - assertThat(coldStats.getIndices(), equalTo(Set.of(coldIndex1.getIndex().getName()))); assertThat(coldStats.getPrimaryShardSizes(), equalTo(List.of(byteSize))); assertThat(coldStats.getTotalShardCount(), is(1)); assertThat(coldStats.getDocCount(), is(coldStats.getTotalShardCount() * docCount)); assertThat(coldStats.getTotalSize(), is(coldStats.getTotalShardCount() * byteSize)); } - - private static DiscoveryNode newNode(int nodeId, DiscoveryNodeRole... roles) { - return DiscoveryNodeUtils.builder("node_" + nodeId).roles(Set.of(roles)).build(); - } - - private static void routeTestShardToNodes( - IndexMetadata index, - int shard, - IndexRoutingTable.Builder indexRoutingTableBuilder, - DiscoveryNode... nodes - ) { - ShardId shardId = new ShardId(index.getIndex(), shard); - IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); - boolean primary = true; - for (DiscoveryNode node : nodes) { - indexShardRoutingBuilder.addShard( - TestShardRouting.newShardRouting(shardId, node.getId(), null, primary, ShardRoutingState.STARTED) - ); - primary = false; - } - indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder); - } - - private NodeIndicesStats buildNodeIndicesStats(RoutingNode routingNode, long bytesPerShard, long docsPerShard) { - Map> indexStats = new HashMap<>(); - for (ShardRouting shardRouting : routingNode) { - ShardId shardId = shardRouting.shardId(); - ShardStats shardStat = shardStat(bytesPerShard, docsPerShard, shardRouting); - IndexShardStats shardStats = new IndexShardStats(shardId, new ShardStats[] { shardStat }); - indexStats.computeIfAbsent(shardId.getIndex(), k -> new ArrayList<>()).add(shardStats); - } - return new NodeIndicesStats(COMMON_STATS, Map.of(), indexStats, true); - } - - private static ShardStats shardStat(long byteCount, long docCount, ShardRouting routing) { - StoreStats storeStats = new StoreStats(randomNonNegativeLong(), byteCount, 0L); - DocsStats docsStats = new DocsStats(docCount, 0L, byteCount); - Path fakePath = PathUtils.get("test/dir/" + routing.shardId().getIndex().getUUID() + "/" + routing.shardId().id()); - ShardPath fakeShardPath = new ShardPath(false, fakePath, fakePath, routing.shardId()); - CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL); - commonStats.getStore().add(storeStats); - commonStats.getDocs().add(docsStats); - return new ShardStats(routing, fakeShardPath, commonStats, null, null, null, false, 0); - } - - private static IndexMetadata indexMetadata(String indexName, int numberOfShards, int numberOfReplicas, String... dataTierPrefs) { - Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), numberOfShards, numberOfReplicas).put( - SETTING_CREATION_DATE, - System.currentTimeMillis() - ); - - if (dataTierPrefs.length > 1) { - StringBuilder tierBuilder = new StringBuilder(dataTierPrefs[0]); - for (int idx = 1; idx < dataTierPrefs.length; idx++) { - tierBuilder.append(',').append(dataTierPrefs[idx]); - } - settingsBuilder.put(DataTier.TIER_PREFERENCE, tierBuilder.toString()); - } else if (dataTierPrefs.length == 1) { - settingsBuilder.put(DataTier.TIER_PREFERENCE, dataTierPrefs[0]); - } - - return IndexMetadata.builder(indexName).settings(settingsBuilder.build()).timestampRange(IndexLongFieldRange.UNKNOWN).build(); - } } From 2baccbf075a816fbd88c24a463c1a2a4f47c6acc Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 10 Nov 2023 15:29:48 +0200 Subject: [PATCH 12/12] Polishing --- .../datatiers/DataTiersUsageTransportAction.java | 8 +++++++- .../NodesDataTiersUsageTransportAction.java | 16 ---------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java index b437108919a89..b5a5e2a4e3273 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.ShardRouting; @@ -95,7 +96,12 @@ static Map> getIndicesGroupedByTier(ClusterState state, List .collect(Collectors.toSet()); Map> indicesByTierPreference = new HashMap<>(); for (String indexName : indices) { - List tierPreference = state.metadata().index(indexName).getTierPreference(); + IndexMetadata indexMetadata = state.metadata().index(indexName); + // If the index was deleted in the meantime, skip + if (indexMetadata == null) { + continue; + } + List tierPreference = indexMetadata.getTierPreference(); if (tierPreference.isEmpty() == false) { indicesByTierPreference.computeIfAbsent(tierPreference.get(0), ignored -> new HashSet<>()).add(indexName); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java index e4d853f4e6050..85b1fa34c2dd4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java @@ -157,18 +157,6 @@ public NodesRequest() { super((String[]) null); } - public NodesRequest(StreamInput in) throws IOException { - super(in); - } - - /** - * Get stats from nodes based on the nodes ids specified. If none are passed, stats - * for all nodes will be returned. - */ - public NodesRequest(String... nodesIds) { - super(nodesIds); - } - @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { return new CancellableTask(id, type, action, "", parentTaskId, headers); @@ -205,10 +193,6 @@ public void writeTo(StreamOutput out) throws IOException { public static class NodesResponse extends BaseNodesResponse { - public NodesResponse(StreamInput in) throws IOException { - super(in); - } - public NodesResponse(ClusterName clusterName, List nodes, List failures) { super(clusterName, nodes, failures); }