diff --git a/docs/changelog/102140.yaml b/docs/changelog/102140.yaml new file mode 100644 index 0000000000000..0f086649b9710 --- /dev/null +++ b/docs/changelog/102140.yaml @@ -0,0 +1,6 @@ +pr: 102140 +summary: Collect data tiers usage stats more efficiently +area: ILM+SLM +type: bug +issues: + - 100230 \ No newline at end of file diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java index 6421b70f9e453..20231af156ee1 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java @@ -26,9 +26,9 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.core.DataTiersFeatureSetUsage; import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder; import org.elasticsearch.xpack.core.action.XPackUsageResponse; +import org.elasticsearch.xpack.core.datatiers.DataTiersFeatureSetUsage; import org.junit.Before; import java.util.ArrayList; diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java index f669bb8589fd7..faeb760b3c181 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.core.rest.action; import org.apache.http.client.methods.HttpGet; -import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; @@ -35,6 +34,7 @@ import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageResponse; +import org.elasticsearch.xpack.core.datatiers.NodesDataTiersUsageTransportAction; import java.nio.file.Path; import java.util.Arrays; @@ -76,7 +76,7 @@ public void testCancellation() throws Exception { final SubscribableListener nodeStatsRequestsReleaseListener = new SubscribableListener<>(); for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { ((MockTransportService) transportService).addRequestHandlingBehavior( - TransportNodesStatsAction.TYPE.name() + "[n]", + NodesDataTiersUsageTransportAction.TYPE.name() + "[n]", (handler, request, channel, task) -> { tasksBlockedLatch.countDown(); nodeStatsRequestsReleaseListener.addListener( @@ -94,14 +94,13 @@ public void testCancellation() throws Exception { safeAwait(tasksBlockedLatch); // must wait for the node-level tasks to start to avoid cancelling being handled earlier cancellable.cancel(); - // NB this test works by blocking node-level stats requests; when #100230 is addressed this will need to target a different action. - assertAllCancellableTasksAreCancelled(TransportNodesStatsAction.TYPE.name()); + assertAllCancellableTasksAreCancelled(NodesDataTiersUsageTransportAction.TYPE.name()); assertAllCancellableTasksAreCancelled(XPackUsageAction.NAME); nodeStatsRequestsReleaseListener.onResponse(null); expectThrows(CancellationException.class, future::actionGet); - assertAllTasksHaveFinished(TransportNodesStatsAction.TYPE.name()); + assertAllTasksHaveFinished(NodesDataTiersUsageTransportAction.TYPE.name()); assertAllTasksHaveFinished(XPackUsageAction.NAME); } diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index deb3c4384a04b..c4c978f656d21 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -57,6 +57,7 @@ exports org.elasticsearch.xpack.core.common.validation; exports org.elasticsearch.xpack.core.common; exports org.elasticsearch.xpack.core.datastreams; + exports org.elasticsearch.xpack.core.datatiers; exports org.elasticsearch.xpack.core.deprecation; exports org.elasticsearch.xpack.core.downsample; exports org.elasticsearch.xpack.core.enrich.action; @@ -226,4 +227,6 @@ with org.elasticsearch.xpack.core.ml.MlConfigVersionComponent, org.elasticsearch.xpack.core.transform.TransformConfigVersionComponent; + + provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.core.XPackFeatures; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java deleted file mode 100644 index 295df1ea51b6b..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.client.internal.ParentTaskAssigningClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.allocation.DataTier; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.protocol.xpack.XPackUsageRequest; -import org.elasticsearch.search.aggregations.metrics.TDigestState; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.StreamSupport; - -public class DataTiersUsageTransportAction extends XPackUsageFeatureTransportAction { - - private final Client client; - - @Inject - public DataTiersUsageTransportAction( - TransportService transportService, - ClusterService clusterService, - ThreadPool threadPool, - ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, - Client client - ) { - super( - XPackUsageFeatureAction.DATA_TIERS.name(), - transportService, - clusterService, - threadPool, - actionFilters, - indexNameExpressionResolver - ); - this.client = client; - } - - @Override - protected void masterOperation( - Task task, - XPackUsageRequest request, - ClusterState state, - ActionListener listener - ) { - new ParentTaskAssigningClient(client, clusterService.localNode(), task).admin() - .cluster() - .prepareNodesStats() - .all() - .setIndices(CommonStatsFlags.ALL) - .execute(listener.delegateFailureAndWrap((delegate, nodesStatsResponse) -> { - final RoutingNodes routingNodes = state.getRoutingNodes(); - final Map indices = state.getMetadata().getIndices(); - - // Determine which tiers each index would prefer to be within - Map indicesToTiers = tierIndices(indices); - - // Generate tier specific stats for the nodes and indices - Map tierSpecificStats = calculateStats( - nodesStatsResponse.getNodes(), - indicesToTiers, - routingNodes - ); - - delegate.onResponse(new XPackUsageFeatureResponse(new DataTiersFeatureSetUsage(tierSpecificStats))); - })); - } - - // Visible for testing - // Takes a registry of indices and returns a mapping of index name to which tier it most prefers. Always 1 to 1, some may filter out. - static Map tierIndices(Map indices) { - Map indexByTier = new HashMap<>(); - indices.entrySet().forEach(entry -> { - String tierPref = entry.getValue().getSettings().get(DataTier.TIER_PREFERENCE); - if (Strings.hasText(tierPref)) { - String[] tiers = tierPref.split(","); - if (tiers.length > 0) { - indexByTier.put(entry.getKey(), tiers[0]); - } - } - }); - return indexByTier; - } - - /** - * Accumulator to hold intermediate data tier stats before final calculation. - */ - private static class TierStatsAccumulator { - int nodeCount = 0; - Set indexNames = new HashSet<>(); - int totalShardCount = 0; - long totalByteCount = 0; - long docCount = 0; - int primaryShardCount = 0; - long primaryByteCount = 0L; - final TDigestState valueSketch = TDigestState.create(1000); - } - - // Visible for testing - static Map calculateStats( - List nodesStats, - Map indexByTier, - RoutingNodes routingNodes - ) { - Map statsAccumulators = new HashMap<>(); - for (NodeStats nodeStats : nodesStats) { - aggregateDataTierNodeCounts(nodeStats, statsAccumulators); - aggregateDataTierIndexStats(nodeStats, routingNodes, indexByTier, statsAccumulators); - } - Map results = new HashMap<>(); - for (Map.Entry entry : statsAccumulators.entrySet()) { - results.put(entry.getKey(), calculateFinalTierStats(entry.getValue())); - } - return results; - } - - /** - * Determine which data tiers this node belongs to (if any), and increment the node counts for those tiers. - */ - private static void aggregateDataTierNodeCounts(NodeStats nodeStats, Map tiersStats) { - nodeStats.getNode() - .getRoles() - .stream() - .map(DiscoveryNodeRole::roleName) - .filter(DataTier::validTierName) - .forEach(tier -> tiersStats.computeIfAbsent(tier, k -> new TierStatsAccumulator()).nodeCount++); - } - - /** - * Locate which indices are hosted on the node specified by the NodeStats, then group and aggregate the available index stats by tier. - */ - private static void aggregateDataTierIndexStats( - NodeStats nodeStats, - RoutingNodes routingNodes, - Map indexByTier, - Map accumulators - ) { - final RoutingNode node = routingNodes.node(nodeStats.getNode().getId()); - if (node != null) { - StreamSupport.stream(node.spliterator(), false) - .map(ShardRouting::index) - .distinct() - .forEach(index -> classifyIndexAndCollectStats(index, nodeStats, indexByTier, node, accumulators)); - } - } - - /** - * Determine which tier an index belongs in, then accumulate its stats into that tier's stats. - */ - private static void classifyIndexAndCollectStats( - Index index, - NodeStats nodeStats, - Map indexByTier, - RoutingNode node, - Map accumulators - ) { - // Look up which tier this index belongs to (its most preferred) - String indexTier = indexByTier.get(index.getName()); - if (indexTier != null) { - final TierStatsAccumulator accumulator = accumulators.computeIfAbsent(indexTier, k -> new TierStatsAccumulator()); - accumulator.indexNames.add(index.getName()); - aggregateDataTierShardStats(nodeStats, index, node, accumulator); - } - } - - /** - * Collect shard-level data tier stats from shard stats contained in the node stats response. - */ - private static void aggregateDataTierShardStats(NodeStats nodeStats, Index index, RoutingNode node, TierStatsAccumulator accumulator) { - // Shard based stats - final List allShardStats = nodeStats.getIndices().getShardStats(index); - if (allShardStats != null) { - for (IndexShardStats shardStat : allShardStats) { - accumulator.totalByteCount += shardStat.getTotal().getStore().totalDataSetSizeInBytes(); - accumulator.docCount += shardStat.getTotal().getDocs().getCount(); - - // Accumulate stats about started shards - ShardRouting shardRouting = node.getByShardId(shardStat.getShardId()); - if (shardRouting != null && shardRouting.state() == ShardRoutingState.STARTED) { - accumulator.totalShardCount += 1; - - // Accumulate stats about started primary shards - StoreStats primaryStoreStats = shardStat.getPrimary().getStore(); - if (primaryStoreStats != null) { - // if primaryStoreStats is null, it means there is no primary on the node in question - accumulator.primaryShardCount++; - long primarySize = primaryStoreStats.totalDataSetSizeInBytes(); - accumulator.primaryByteCount += primarySize; - accumulator.valueSketch.add(primarySize); - } - } - } - } - } - - private static DataTiersFeatureSetUsage.TierSpecificStats calculateFinalTierStats(TierStatsAccumulator accumulator) { - long primaryShardSizeMedian = (long) accumulator.valueSketch.quantile(0.5); - long primaryShardSizeMAD = computeMedianAbsoluteDeviation(accumulator.valueSketch); - return new DataTiersFeatureSetUsage.TierSpecificStats( - accumulator.nodeCount, - accumulator.indexNames.size(), - accumulator.totalShardCount, - accumulator.primaryShardCount, - accumulator.docCount, - accumulator.totalByteCount, - accumulator.primaryByteCount, - primaryShardSizeMedian, - primaryShardSizeMAD - ); - } - - // Visible for testing - static long computeMedianAbsoluteDeviation(TDigestState valuesSketch) { - if (valuesSketch.size() == 0) { - return 0; - } else { - final double approximateMedian = valuesSketch.quantile(0.5); - final TDigestState approximatedDeviationsSketch = TDigestState.createUsingParamsFrom(valuesSketch); - valuesSketch.centroids().forEach(centroid -> { - final double deviation = Math.abs(approximateMedian - centroid.mean()); - approximatedDeviationsSketch.add(deviation, centroid.count()); - }); - - return (long) approximatedDeviationsSketch.quantile(0.5); - } - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 6d019e50f9d5f..ac16631bacb73 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage; import org.elasticsearch.xpack.core.datastreams.DataStreamLifecycleFeatureSetUsage; +import org.elasticsearch.xpack.core.datatiers.DataTiersFeatureSetUsage; import org.elasticsearch.xpack.core.downsample.DownsampleShardStatus; import org.elasticsearch.xpack.core.enrich.EnrichFeatureSetUsage; import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyStatus; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatures.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatures.java new file mode 100644 index 0000000000000..97934cbda09ab --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatures.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.xpack.core.datatiers.NodesDataTiersUsageTransportAction; + +import java.util.Set; + +/** + * Provides the XPack features that this version of the code supports + */ +public class XPackFeatures implements FeatureSpecification { + + @Override + public Set getFeatures() { + return Set.of( + NodesDataTiersUsageTransportAction.LOCALLY_PRECALCULATED_STATS_FEATURE // Added in 8.12 + ); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index d02e3f43d80cb..66534cccff064 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -98,6 +98,9 @@ import org.elasticsearch.xpack.core.action.XPackUsageResponse; import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.async.TransportDeleteAsyncResultAction; +import org.elasticsearch.xpack.core.datatiers.DataTiersInfoTransportAction; +import org.elasticsearch.xpack.core.datatiers.DataTiersUsageTransportAction; +import org.elasticsearch.xpack.core.datatiers.NodesDataTiersUsageTransportAction; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction; import org.elasticsearch.xpack.core.rest.action.RestXPackUsageAction; @@ -362,6 +365,7 @@ public Collection createComponents(PluginServices services) { actions.add(new ActionHandler<>(XPackUsageFeatureAction.DATA_STREAM_LIFECYCLE, DataStreamLifecycleUsageTransportAction.class)); actions.add(new ActionHandler<>(XPackUsageFeatureAction.HEALTH, HealthApiUsageTransportAction.class)); actions.add(new ActionHandler<>(XPackUsageFeatureAction.REMOTE_CLUSTERS, RemoteClusterUsageTransportAction.class)); + actions.add(new ActionHandler<>(NodesDataTiersUsageTransportAction.TYPE, NodesDataTiersUsageTransportAction.class)); return actions; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsage.java similarity index 98% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsage.java index 0bf21f66b4888..f990118763bad 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsage.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core; +package org.elasticsearch.xpack.core.datatiers; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -16,6 +16,8 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; import java.io.IOException; import java.util.Collections; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersInfoTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java similarity index 91% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersInfoTransportAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java index 6134813dc4651..3af1945c53d3f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersInfoTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java @@ -5,11 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.core; +package org.elasticsearch.xpack.core.datatiers; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java new file mode 100644 index 0000000000000..728309926302a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.ParentTaskAssigningClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class DataTiersUsageTransportAction extends XPackUsageFeatureTransportAction { + + private final Client client; + private final FeatureService featureService; + + @Inject + public DataTiersUsageTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + Client client, + FeatureService featureService + ) { + super( + XPackUsageFeatureAction.DATA_TIERS.name(), + transportService, + clusterService, + threadPool, + actionFilters, + indexNameExpressionResolver + ); + this.client = client; + this.featureService = featureService; + } + + @Override + protected void masterOperation( + Task task, + XPackUsageRequest request, + ClusterState state, + ActionListener listener + ) { + if (featureService.clusterHasFeature(state, NodesDataTiersUsageTransportAction.LOCALLY_PRECALCULATED_STATS_FEATURE)) { + new ParentTaskAssigningClient(client, clusterService.localNode(), task).admin() + .cluster() + .execute( + NodesDataTiersUsageTransportAction.TYPE, + new NodesDataTiersUsageTransportAction.NodesRequest(), + listener.delegateFailureAndWrap((delegate, response) -> { + // Generate tier specific stats for the nodes and indices + delegate.onResponse( + new XPackUsageFeatureResponse( + new DataTiersFeatureSetUsage( + aggregateStats(response.getNodes(), getIndicesGroupedByTier(state, response.getNodes())) + ) + ) + ); + }) + ); + } else { + new ParentTaskAssigningClient(client, clusterService.localNode(), task).admin() + .cluster() + .prepareNodesStats() + .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Docs, CommonStatsFlags.Flag.Store)) + .execute(listener.delegateFailureAndWrap((delegate, nodesStatsResponse) -> { + List response = nodesStatsResponse.getNodes() + .stream() + .map( + nodeStats -> new NodeDataTiersUsage(nodeStats.getNode(), precalculateLocalStatsFromNodeStats(nodeStats, state)) + ) + .toList(); + delegate.onResponse( + new XPackUsageFeatureResponse( + new DataTiersFeatureSetUsage(aggregateStats(response, getIndicesGroupedByTier(state, response))) + ) + ); + })); + } + } + + // Visible for testing + static Map> getIndicesGroupedByTier(ClusterState state, List nodes) { + Set indices = nodes.stream() + .map(nodeResponse -> state.getRoutingNodes().node(nodeResponse.getNode().getId())) + .filter(Objects::nonNull) + .flatMap(node -> StreamSupport.stream(node.spliterator(), false)) + .map(ShardRouting::getIndexName) + .collect(Collectors.toSet()); + Map> indicesByTierPreference = new HashMap<>(); + for (String indexName : indices) { + IndexMetadata indexMetadata = state.metadata().index(indexName); + // If the index was deleted in the meantime, skip + if (indexMetadata == null) { + continue; + } + List tierPreference = indexMetadata.getTierPreference(); + if (tierPreference.isEmpty() == false) { + indicesByTierPreference.computeIfAbsent(tierPreference.get(0), ignored -> new HashSet<>()).add(indexName); + } + } + return indicesByTierPreference; + } + + /** + * Accumulator to hold intermediate data tier stats before final calculation. + */ + private static class TierStatsAccumulator { + int nodeCount = 0; + Set indexNames = new HashSet<>(); + int totalShardCount = 0; + long totalByteCount = 0; + long docCount = 0; + int primaryShardCount = 0; + long primaryByteCount = 0L; + final TDigestState valueSketch = TDigestState.create(1000); + } + + // Visible for testing + static Map aggregateStats( + List nodeDataTiersUsages, + Map> tierPreference + ) { + Map statsAccumulators = new HashMap<>(); + for (String tier : tierPreference.keySet()) { + statsAccumulators.put(tier, new TierStatsAccumulator()); + statsAccumulators.get(tier).indexNames.addAll(tierPreference.get(tier)); + } + for (NodeDataTiersUsage nodeDataTiersUsage : nodeDataTiersUsages) { + aggregateDataTierNodeCounts(nodeDataTiersUsage, statsAccumulators); + aggregateDataTierIndexStats(nodeDataTiersUsage, statsAccumulators); + } + Map results = new HashMap<>(); + for (Map.Entry entry : statsAccumulators.entrySet()) { + results.put(entry.getKey(), aggregateFinalTierStats(entry.getValue())); + } + return results; + } + + /** + * Determine which data tiers each node belongs to (if any), and increment the node counts for those tiers. + */ + private static void aggregateDataTierNodeCounts(NodeDataTiersUsage nodeStats, Map tiersStats) { + nodeStats.getNode() + .getRoles() + .stream() + .map(DiscoveryNodeRole::roleName) + .filter(DataTier::validTierName) + .forEach(tier -> tiersStats.computeIfAbsent(tier, k -> new TierStatsAccumulator()).nodeCount++); + } + + /** + * Iterate the preferred tiers of the indices for a node and aggregate their stats. + */ + private static void aggregateDataTierIndexStats(NodeDataTiersUsage nodeDataTiersUsage, Map accumulators) { + for (Map.Entry entry : nodeDataTiersUsage.getUsageStatsByTier().entrySet()) { + String tier = entry.getKey(); + NodeDataTiersUsage.UsageStats usage = entry.getValue(); + if (DataTier.validTierName(tier)) { + TierStatsAccumulator accumulator = accumulators.computeIfAbsent(tier, k -> new TierStatsAccumulator()); + accumulator.docCount += usage.getDocCount(); + accumulator.totalByteCount += usage.getTotalSize(); + accumulator.totalShardCount += usage.getTotalShardCount(); + for (Long primaryShardSize : usage.getPrimaryShardSizes()) { + accumulator.primaryShardCount += 1; + accumulator.primaryByteCount += primaryShardSize; + accumulator.valueSketch.add(primaryShardSize); + } + } + } + } + + private static DataTiersFeatureSetUsage.TierSpecificStats aggregateFinalTierStats(TierStatsAccumulator accumulator) { + long primaryShardSizeMedian = (long) accumulator.valueSketch.quantile(0.5); + long primaryShardSizeMAD = computeMedianAbsoluteDeviation(accumulator.valueSketch); + return new DataTiersFeatureSetUsage.TierSpecificStats( + accumulator.nodeCount, + accumulator.indexNames.size(), + accumulator.totalShardCount, + accumulator.primaryShardCount, + accumulator.docCount, + accumulator.totalByteCount, + accumulator.primaryByteCount, + primaryShardSizeMedian, + primaryShardSizeMAD + ); + } + + // Visible for testing + static long computeMedianAbsoluteDeviation(TDigestState valuesSketch) { + if (valuesSketch.size() == 0) { + return 0; + } else { + final double approximateMedian = valuesSketch.quantile(0.5); + final TDigestState approximatedDeviationsSketch = TDigestState.createUsingParamsFrom(valuesSketch); + valuesSketch.centroids().forEach(centroid -> { + final double deviation = Math.abs(approximateMedian - centroid.mean()); + approximatedDeviationsSketch.add(deviation, centroid.count()); + }); + + return (long) approximatedDeviationsSketch.quantile(0.5); + } + } + + /** + * In this method we use {@link NodesDataTiersUsageTransportAction#aggregateStats(RoutingNode, Metadata, NodeIndicesStats)} + * to precalculate the stats we need from {@link NodeStats} just like we do in NodesDataTiersUsageTransportAction. + * This way we can be backwards compatible without duplicating the calculation. This is only meant to be used to be + * backwards compatible and it should be removed afterwords. + */ + private static Map precalculateLocalStatsFromNodeStats(NodeStats nodeStats, ClusterState state) { + RoutingNode routingNode = state.getRoutingNodes().node(nodeStats.getNode().getId()); + if (routingNode == null) { + return Map.of(); + } + + return NodesDataTiersUsageTransportAction.aggregateStats(routingNode, state.metadata(), nodeStats.getIndices()); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java new file mode 100644 index 0000000000000..c1903a2910629 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Data tier usage statistics on a specific node. The statistics groups the indices, shard sizes, shard counts based + * on their tier preference. + */ +public class NodeDataTiersUsage extends BaseNodeResponse { + + private final Map usageStatsByTier; + + public static class UsageStats implements Writeable { + private final List primaryShardSizes; + private int totalShardCount; + private long docCount; + private long totalSize; + + public UsageStats() { + this.primaryShardSizes = new ArrayList<>(); + this.totalShardCount = 0; + this.docCount = 0; + this.totalSize = 0; + } + + public UsageStats(List primaryShardSizes, int totalShardCount, long docCount, long totalSize) { + this.primaryShardSizes = primaryShardSizes; + this.totalShardCount = totalShardCount; + this.docCount = docCount; + this.totalSize = totalSize; + } + + static UsageStats read(StreamInput in) throws IOException { + return new UsageStats(in.readCollectionAsList(StreamInput::readVLong), in.readVInt(), in.readVLong(), in.readVLong()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(primaryShardSizes, StreamOutput::writeVLong); + out.writeVInt(totalShardCount); + out.writeVLong(docCount); + out.writeVLong(totalSize); + } + + public void addPrimaryShardSize(long primaryShardSize) { + primaryShardSizes.add(primaryShardSize); + } + + public void incrementTotalSize(long totalSize) { + this.totalSize += totalSize; + } + + public void incrementDocCount(long docCount) { + this.docCount += docCount; + } + + public void incrementTotalShardCount(int totalShardCount) { + this.totalShardCount += totalShardCount; + } + + public List getPrimaryShardSizes() { + return primaryShardSizes; + } + + public int getTotalShardCount() { + return totalShardCount; + } + + public long getDocCount() { + return docCount; + } + + public long getTotalSize() { + return totalSize; + } + } + + public NodeDataTiersUsage(StreamInput in) throws IOException { + super(in); + usageStatsByTier = in.readMap(UsageStats::read); + } + + public NodeDataTiersUsage(DiscoveryNode node, Map usageStatsByTier) { + super(node); + this.usageStatsByTier = usageStatsByTier; + } + + public Map getUsageStatsByTier() { + return Map.copyOf(usageStatsByTier); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeMap(usageStatsByTier, (o, v) -> v.writeTo(o)); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java new file mode 100644 index 0000000000000..06a3b47d47a65 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Sources locally data tier usage stats mainly indices and shard sizes grouped by preferred data tier. + */ +public class NodesDataTiersUsageTransportAction extends TransportNodesAction< + NodesDataTiersUsageTransportAction.NodesRequest, + NodesDataTiersUsageTransportAction.NodesResponse, + NodesDataTiersUsageTransportAction.NodeRequest, + NodeDataTiersUsage> { + + public static final ActionType TYPE = ActionType.localOnly("cluster:monitor/nodes/data_tier_usage"); + public static final NodeFeature LOCALLY_PRECALCULATED_STATS_FEATURE = new NodeFeature("usage.data_tiers.precalculate_stats"); + + private static final CommonStatsFlags STATS_FLAGS = new CommonStatsFlags().clear() + .set(CommonStatsFlags.Flag.Docs, true) + .set(CommonStatsFlags.Flag.Store, true); + + private final IndicesService indicesService; + + @Inject + public NodesDataTiersUsageTransportAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + IndicesService indicesService, + ActionFilters actionFilters + ) { + super( + TYPE.name(), + clusterService, + transportService, + actionFilters, + NodeRequest::new, + threadPool.executor(ThreadPool.Names.MANAGEMENT) + ); + this.indicesService = indicesService; + } + + @Override + protected NodesResponse newResponse(NodesRequest request, List responses, List failures) { + return new NodesResponse(clusterService.getClusterName(), responses, failures); + } + + @Override + protected NodeRequest newNodeRequest(NodesRequest request) { + return NodeRequest.INSTANCE; + } + + @Override + protected NodeDataTiersUsage newNodeResponse(StreamInput in, DiscoveryNode node) throws IOException { + return new NodeDataTiersUsage(in); + } + + @Override + protected NodeDataTiersUsage nodeOperation(NodeRequest nodeRequest, Task task) { + assert task instanceof CancellableTask; + + DiscoveryNode localNode = clusterService.localNode(); + NodeIndicesStats nodeIndicesStats = indicesService.stats(STATS_FLAGS, true); + ClusterState state = clusterService.state(); + RoutingNode routingNode = state.getRoutingNodes().node(localNode.getId()); + Map usageStatsByTier = aggregateStats(routingNode, state.metadata(), nodeIndicesStats); + return new NodeDataTiersUsage(clusterService.localNode(), usageStatsByTier); + } + + // For bwc & testing purposes + static Map aggregateStats( + RoutingNode routingNode, + Metadata metadata, + NodeIndicesStats nodeIndicesStats + ) { + if (routingNode == null) { + return Map.of(); + } + Map usageStatsByTier = new HashMap<>(); + Set localIndices = StreamSupport.stream(routingNode.spliterator(), false) + .map(routing -> routing.index().getName()) + .collect(Collectors.toSet()); + for (String indexName : localIndices) { + IndexMetadata indexMetadata = metadata.index(indexName); + if (indexMetadata == null) { + continue; + } + String tier = indexMetadata.getTierPreference().isEmpty() ? null : indexMetadata.getTierPreference().get(0); + if (tier != null) { + NodeDataTiersUsage.UsageStats usageStats = usageStatsByTier.computeIfAbsent( + tier, + ignored -> new NodeDataTiersUsage.UsageStats() + ); + List allShardStats = nodeIndicesStats.getShardStats(indexMetadata.getIndex()); + if (allShardStats != null) { + for (IndexShardStats indexShardStats : allShardStats) { + usageStats.incrementTotalSize(indexShardStats.getTotal().getStore().totalDataSetSizeInBytes()); + usageStats.incrementDocCount(indexShardStats.getTotal().getDocs().getCount()); + + ShardRouting shardRouting = routingNode.getByShardId(indexShardStats.getShardId()); + if (shardRouting != null && shardRouting.state() == ShardRoutingState.STARTED) { + usageStats.incrementTotalShardCount(1); + + // Accumulate stats about started primary shards + StoreStats primaryStoreStats = indexShardStats.getPrimary().getStore(); + if (shardRouting.primary() && primaryStoreStats != null) { + usageStats.addPrimaryShardSize(primaryStoreStats.totalDataSetSizeInBytes()); + } + } + } + } + } + } + return usageStatsByTier; + } + + public static class NodesRequest extends BaseNodesRequest { + + public NodesRequest() { + super((String[]) null); + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + } + + public static class NodeRequest extends TransportRequest { + + static final NodeRequest INSTANCE = new NodeRequest(); + + public NodeRequest(StreamInput in) throws IOException { + super(in); + } + + public NodeRequest() { + + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + } + + public static class NodesResponse extends BaseNodesResponse { + + public NodesResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readCollectionAsList(NodeDataTiersUsage::new); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeCollection(nodes); + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/core/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification new file mode 100644 index 0000000000000..545918cbab502 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -0,0 +1,8 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +org.elasticsearch.xpack.core.XPackFeatures diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersUsageTransportActionTests.java deleted file mode 100644 index 93e991b0fa5ae..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersUsageTransportActionTests.java +++ /dev/null @@ -1,786 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core; - -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.ShardStats; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.cluster.node.DiscoveryNodeUtils; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; -import org.elasticsearch.cluster.routing.allocation.DataTier; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.shard.DocsStats; -import org.elasticsearch.index.shard.IndexLongFieldRange; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.ShardPath; -import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.indices.NodeIndicesStats; -import org.elasticsearch.search.aggregations.metrics.TDigestState; -import org.elasticsearch.test.ESTestCase; - -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class DataTiersUsageTransportActionTests extends ESTestCase { - - public void testCalculateMAD() { - assertThat(DataTiersUsageTransportAction.computeMedianAbsoluteDeviation(TDigestState.create(10)), equalTo(0L)); - - TDigestState sketch = TDigestState.create(randomDoubleBetween(1, 1000, false)); - sketch.add(1); - sketch.add(1); - sketch.add(2); - sketch.add(2); - sketch.add(4); - sketch.add(6); - sketch.add(9); - assertThat(DataTiersUsageTransportAction.computeMedianAbsoluteDeviation(sketch), equalTo(1L)); - } - - public void testTierIndices() { - IndexMetadata hotIndex1 = indexMetadata("hot-1", 1, 0, DataTier.DATA_HOT); - IndexMetadata hotIndex2 = indexMetadata("hot-2", 1, 0, DataTier.DATA_HOT); - IndexMetadata warmIndex1 = indexMetadata("warm-1", 1, 0, DataTier.DATA_WARM); - IndexMetadata coldIndex1 = indexMetadata("cold-1", 1, 0, DataTier.DATA_COLD); - IndexMetadata coldIndex2 = indexMetadata("cold-2", 1, 0, DataTier.DATA_COLD, DataTier.DATA_WARM); // Prefers cold over warm - IndexMetadata nonTiered = indexMetadata("non-tier", 1, 0); // No tier - - Map indices = new HashMap<>(); - indices.put("hot-1", hotIndex1); - indices.put("hot-2", hotIndex2); - indices.put("warm-1", warmIndex1); - indices.put("cold-1", coldIndex1); - indices.put("cold-2", coldIndex2); - indices.put("non-tier", nonTiered); - - Map tiers = DataTiersUsageTransportAction.tierIndices(indices); - assertThat(tiers.size(), equalTo(5)); - assertThat(tiers.get("hot-1"), equalTo(DataTier.DATA_HOT)); - assertThat(tiers.get("hot-2"), equalTo(DataTier.DATA_HOT)); - assertThat(tiers.get("warm-1"), equalTo(DataTier.DATA_WARM)); - assertThat(tiers.get("cold-1"), equalTo(DataTier.DATA_COLD)); - assertThat(tiers.get("cold-2"), equalTo(DataTier.DATA_COLD)); - assertThat(tiers.get("non-tier"), nullValue()); - } - - public void testCalculateStatsNoTiers() { - // Nodes: 0 Tiered Nodes, 1 Data Node - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); - DiscoveryNode leader = newNode(0, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - - DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode1); - - discoBuilder.localNodeId(dataNode1.getId()); - - // Indices: 1 Regular index - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata index1 = indexMetadata("index_1", 3, 1); - metadataBuilder.put(index1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index1.getIndex()); - routeTestShardToNodes(index1, 0, indexRoutingTableBuilder, dataNode1); - routeTestShardToNodes(index1, 1, indexRoutingTableBuilder, dataNode1); - routeTestShardToNodes(index1, 2, indexRoutingTableBuilder, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); - - // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() - ); - - // Verify - No results when no tiers present - assertThat(tierSpecificStats.size(), is(0)); - } - - public void testCalculateStatsTieredNodesOnly() { - // Nodes: 1 Data, 1 Hot, 1 Warm, 1 Cold, 1 Frozen - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); - DiscoveryNode leader = newNode(0, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - - DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode1); - DiscoveryNode hotNode1 = newNode(2, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode1); - DiscoveryNode warmNode1 = newNode(3, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode1); - DiscoveryNode coldNode1 = newNode(4, DiscoveryNodeRole.DATA_COLD_NODE_ROLE); - discoBuilder.add(coldNode1); - DiscoveryNode frozenNode1 = newNode(5, DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE); - discoBuilder.add(frozenNode1); - - discoBuilder.localNodeId(dataNode1.getId()); - - // Indices: 1 Regular index, not hosted on any tiers - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata index1 = indexMetadata("index_1", 3, 1); - metadataBuilder.put(index1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index1.getIndex()); - routeTestShardToNodes(index1, 0, indexRoutingTableBuilder, dataNode1); - routeTestShardToNodes(index1, 1, indexRoutingTableBuilder, dataNode1); - routeTestShardToNodes(index1, 2, indexRoutingTableBuilder, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); - - // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() - ); - - // Verify - Results are present but they lack index numbers because none are tiered - assertThat(tierSpecificStats.size(), is(4)); - - DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); - assertThat(hotStats, is(notNullValue())); - assertThat(hotStats.nodeCount, is(1)); - assertThat(hotStats.indexCount, is(0)); - assertThat(hotStats.totalShardCount, is(0)); - assertThat(hotStats.docCount, is(0L)); - assertThat(hotStats.totalByteCount, is(0L)); - assertThat(hotStats.primaryShardCount, is(0)); - assertThat(hotStats.primaryByteCount, is(0L)); - assertThat(hotStats.primaryByteCountMedian, is(0L)); // All same size - assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size - - DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); - assertThat(warmStats, is(notNullValue())); - assertThat(warmStats.nodeCount, is(1)); - assertThat(warmStats.indexCount, is(0)); - assertThat(warmStats.totalShardCount, is(0)); - assertThat(warmStats.docCount, is(0L)); - assertThat(warmStats.totalByteCount, is(0L)); - assertThat(warmStats.primaryShardCount, is(0)); - assertThat(warmStats.primaryByteCount, is(0L)); - assertThat(warmStats.primaryByteCountMedian, is(0L)); // All same size - assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size - - DataTiersFeatureSetUsage.TierSpecificStats coldStats = tierSpecificStats.get(DataTier.DATA_COLD); - assertThat(coldStats, is(notNullValue())); - assertThat(coldStats.nodeCount, is(1)); - assertThat(coldStats.indexCount, is(0)); - assertThat(coldStats.totalShardCount, is(0)); - assertThat(coldStats.docCount, is(0L)); - assertThat(coldStats.totalByteCount, is(0L)); - assertThat(coldStats.primaryShardCount, is(0)); - assertThat(coldStats.primaryByteCount, is(0L)); - assertThat(coldStats.primaryByteCountMedian, is(0L)); // All same size - assertThat(coldStats.primaryShardBytesMAD, is(0L)); // All same size - - DataTiersFeatureSetUsage.TierSpecificStats frozenStats = tierSpecificStats.get(DataTier.DATA_FROZEN); - assertThat(frozenStats, is(notNullValue())); - assertThat(frozenStats.nodeCount, is(1)); - assertThat(frozenStats.indexCount, is(0)); - assertThat(frozenStats.totalShardCount, is(0)); - assertThat(frozenStats.docCount, is(0L)); - assertThat(frozenStats.totalByteCount, is(0L)); - assertThat(frozenStats.primaryShardCount, is(0)); - assertThat(frozenStats.primaryByteCount, is(0L)); - assertThat(frozenStats.primaryByteCountMedian, is(0L)); // All same size - assertThat(frozenStats.primaryShardBytesMAD, is(0L)); // All same size - } - - public void testCalculateStatsTieredIndicesOnly() { - // Nodes: 3 Data, 0 Tiered - Only hosting indices on generic data nodes - int nodeId = 0; - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); - DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - - DiscoveryNode dataNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode1); - DiscoveryNode dataNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode2); - DiscoveryNode dataNode3 = newNode(nodeId, DiscoveryNodeRole.DATA_ROLE); - discoBuilder.add(dataNode3); - - discoBuilder.localNodeId(dataNode1.getId()); - - // Indices: 1 Hot index, 2 Warm indices, 3 Cold indices - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); - metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); - routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, dataNode1, dataNode2); - routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, dataNode2, dataNode3); - routeTestShardToNodes(hotIndex1, 2, indexRoutingTableBuilder, dataNode3, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); - routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, dataNode1, dataNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata warmIndex2 = indexMetadata("warm_index_2", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex2.getIndex()); - routeTestShardToNodes(warmIndex2, 0, indexRoutingTableBuilder, dataNode3, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata coldIndex1 = indexMetadata("cold_index_1", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex1.getIndex()); - routeTestShardToNodes(coldIndex1, 0, indexRoutingTableBuilder, dataNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata coldIndex2 = indexMetadata("cold_index_2", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex2.getIndex()); - routeTestShardToNodes(coldIndex2, 0, indexRoutingTableBuilder, dataNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata coldIndex3 = indexMetadata("cold_index_3", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex3, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex3.getIndex()); - routeTestShardToNodes(coldIndex3, 0, indexRoutingTableBuilder, dataNode3); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); - - // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() - ); - - // Verify - Index stats exist for the tiers, but no tiered nodes are found - assertThat(tierSpecificStats.size(), is(3)); - - DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); - assertThat(hotStats, is(notNullValue())); - assertThat(hotStats.nodeCount, is(0)); - assertThat(hotStats.indexCount, is(1)); - assertThat(hotStats.totalShardCount, is(6)); - assertThat(hotStats.docCount, is(6 * docCount)); - assertThat(hotStats.totalByteCount, is(6 * byteSize)); - assertThat(hotStats.primaryShardCount, is(3)); - assertThat(hotStats.primaryByteCount, is(3 * byteSize)); - assertThat(hotStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size - - DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); - assertThat(warmStats, is(notNullValue())); - assertThat(warmStats.nodeCount, is(0)); - assertThat(warmStats.indexCount, is(2)); - assertThat(warmStats.totalShardCount, is(4)); - assertThat(warmStats.docCount, is(4 * docCount)); - assertThat(warmStats.totalByteCount, is(4 * byteSize)); - assertThat(warmStats.primaryShardCount, is(2)); - assertThat(warmStats.primaryByteCount, is(2 * byteSize)); - assertThat(warmStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size - - DataTiersFeatureSetUsage.TierSpecificStats coldStats = tierSpecificStats.get(DataTier.DATA_COLD); - assertThat(coldStats, is(notNullValue())); - assertThat(coldStats.nodeCount, is(0)); - assertThat(coldStats.indexCount, is(3)); - assertThat(coldStats.totalShardCount, is(3)); - assertThat(coldStats.docCount, is(3 * docCount)); - assertThat(coldStats.totalByteCount, is(3 * byteSize)); - assertThat(coldStats.primaryShardCount, is(3)); - assertThat(coldStats.primaryByteCount, is(3 * byteSize)); - assertThat(coldStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(coldStats.primaryShardBytesMAD, is(0L)); // All same size - } - - public void testCalculateStatsReasonableCase() { - // Nodes: 3 Hot, 5 Warm, 1 Cold - int nodeId = 0; - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); - DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - - DiscoveryNode hotNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode1); - DiscoveryNode hotNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode2); - DiscoveryNode hotNode3 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode3); - DiscoveryNode warmNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode1); - DiscoveryNode warmNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode2); - DiscoveryNode warmNode3 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode3); - DiscoveryNode warmNode4 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode4); - DiscoveryNode warmNode5 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(warmNode5); - DiscoveryNode coldNode1 = newNode(nodeId, DiscoveryNodeRole.DATA_COLD_NODE_ROLE); - discoBuilder.add(coldNode1); - - discoBuilder.localNodeId(hotNode1.getId()); - - // Indices: 1 Hot index, 2 Warm indices, 3 Cold indices - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); - metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); - routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, hotNode1, hotNode2); - routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, hotNode2, hotNode3); - routeTestShardToNodes(hotIndex1, 2, indexRoutingTableBuilder, hotNode3, hotNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); - routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, warmNode1, warmNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata warmIndex2 = indexMetadata("warm_index_2", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex2.getIndex()); - routeTestShardToNodes(warmIndex2, 0, indexRoutingTableBuilder, warmNode3, warmNode4); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata coldIndex1 = indexMetadata("cold_index_1", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex1.getIndex()); - routeTestShardToNodes(coldIndex1, 0, indexRoutingTableBuilder, coldNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata coldIndex2 = indexMetadata("cold_index_2", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex2.getIndex()); - routeTestShardToNodes(coldIndex2, 0, indexRoutingTableBuilder, coldNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata coldIndex3 = indexMetadata("cold_index_3", 1, 0, DataTier.DATA_COLD); - metadataBuilder.put(coldIndex3, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex3.getIndex()); - routeTestShardToNodes(coldIndex3, 0, indexRoutingTableBuilder, coldNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); - - // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() - ); - - // Verify - Node and Index stats are both collected - assertThat(tierSpecificStats.size(), is(3)); - - DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); - assertThat(hotStats, is(notNullValue())); - assertThat(hotStats.nodeCount, is(3)); - assertThat(hotStats.indexCount, is(1)); - assertThat(hotStats.totalShardCount, is(6)); - assertThat(hotStats.docCount, is(6 * docCount)); - assertThat(hotStats.totalByteCount, is(6 * byteSize)); - assertThat(hotStats.primaryShardCount, is(3)); - assertThat(hotStats.primaryByteCount, is(3 * byteSize)); - assertThat(hotStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size - - DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); - assertThat(warmStats, is(notNullValue())); - assertThat(warmStats.nodeCount, is(5)); - assertThat(warmStats.indexCount, is(2)); - assertThat(warmStats.totalShardCount, is(4)); - assertThat(warmStats.docCount, is(4 * docCount)); - assertThat(warmStats.totalByteCount, is(4 * byteSize)); - assertThat(warmStats.primaryShardCount, is(2)); - assertThat(warmStats.primaryByteCount, is(2 * byteSize)); - assertThat(warmStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size - - DataTiersFeatureSetUsage.TierSpecificStats coldStats = tierSpecificStats.get(DataTier.DATA_COLD); - assertThat(coldStats, is(notNullValue())); - assertThat(coldStats.nodeCount, is(1)); - assertThat(coldStats.indexCount, is(3)); - assertThat(coldStats.totalShardCount, is(3)); - assertThat(coldStats.docCount, is(3 * docCount)); - assertThat(coldStats.totalByteCount, is(3 * byteSize)); - assertThat(coldStats.primaryShardCount, is(3)); - assertThat(coldStats.primaryByteCount, is(3 * byteSize)); - assertThat(coldStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(coldStats.primaryShardBytesMAD, is(0L)); // All same size - } - - public void testCalculateStatsMixedTiers() { - // Nodes: 3 Hot+Warm - Nodes that are marked as part of multiple tiers - int nodeId = 0; - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); - DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - - DiscoveryNode mixedNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(mixedNode1); - DiscoveryNode mixedNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(mixedNode2); - DiscoveryNode mixedNode3 = newNode(nodeId, DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); - discoBuilder.add(mixedNode3); - - discoBuilder.localNodeId(mixedNode1.getId()); - - // Indices: 1 Hot index, 2 Warm indices - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); - metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); - routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, mixedNode1, mixedNode2); - routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, mixedNode3, mixedNode1); - routeTestShardToNodes(hotIndex1, 2, indexRoutingTableBuilder, mixedNode2, mixedNode3); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); - routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, mixedNode1, mixedNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - IndexMetadata warmIndex2 = indexMetadata("warm_index_2", 1, 1, DataTier.DATA_WARM); - metadataBuilder.put(warmIndex2, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex2.getIndex()); - routeTestShardToNodes(warmIndex2, 0, indexRoutingTableBuilder, mixedNode3, mixedNode1); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); - - // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() - ); - - // Verify - Index stats are separated by their preferred tier, instead of counted - // toward multiple tiers based on their current routing. Nodes are counted for each tier they are in. - assertThat(tierSpecificStats.size(), is(2)); - - DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); - assertThat(hotStats, is(notNullValue())); - assertThat(hotStats.nodeCount, is(3)); - assertThat(hotStats.indexCount, is(1)); - assertThat(hotStats.totalShardCount, is(6)); - assertThat(hotStats.docCount, is(6 * docCount)); - assertThat(hotStats.totalByteCount, is(6 * byteSize)); - assertThat(hotStats.primaryShardCount, is(3)); - assertThat(hotStats.primaryByteCount, is(3 * byteSize)); - assertThat(hotStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size - - DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); - assertThat(warmStats, is(notNullValue())); - assertThat(warmStats.nodeCount, is(3)); - assertThat(warmStats.indexCount, is(2)); - assertThat(warmStats.totalShardCount, is(4)); - assertThat(warmStats.docCount, is(4 * docCount)); - assertThat(warmStats.totalByteCount, is(4 * byteSize)); - assertThat(warmStats.primaryShardCount, is(2)); - assertThat(warmStats.primaryByteCount, is(2 * byteSize)); - assertThat(warmStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size - } - - public void testCalculateStatsStuckInWrongTier() { - // Nodes: 3 Hot, 0 Warm - Emulating indices stuck on non-preferred tiers - int nodeId = 0; - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); - DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); - discoBuilder.add(leader); - discoBuilder.masterNodeId(leader.getId()); - - DiscoveryNode hotNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode1); - DiscoveryNode hotNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode2); - DiscoveryNode hotNode3 = newNode(nodeId, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); - discoBuilder.add(hotNode3); - - discoBuilder.localNodeId(hotNode1.getId()); - - // Indices: 1 Hot index, 1 Warm index (Warm index is allocated to less preferred hot node because warm nodes are missing) - Metadata.Builder metadataBuilder = Metadata.builder(); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); - - IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); - metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); - routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, hotNode1, hotNode2); - routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, hotNode3, hotNode1); - routeTestShardToNodes(hotIndex1, 2, indexRoutingTableBuilder, hotNode2, hotNode3); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM, DataTier.DATA_HOT); - metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); - { - IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); - routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, hotNode1, hotNode2); - routingTableBuilder.add(indexRoutingTableBuilder.build()); - } - - // Cluster State and create stats responses - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .nodes(discoBuilder) - .metadata(metadataBuilder) - .routingTable(routingTableBuilder.build()) - .build(); - - long byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB - long docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million - List nodeStatsList = buildNodeStats(clusterState, byteSize, docCount); - - // Calculate usage - Map indexByTier = DataTiersUsageTransportAction.tierIndices(clusterState.metadata().indices()); - Map tierSpecificStats = DataTiersUsageTransportAction.calculateStats( - nodeStatsList, - indexByTier, - clusterState.getRoutingNodes() - ); - - // Verify - Warm indices are still calculated separately from Hot ones, despite Warm nodes missing - assertThat(tierSpecificStats.size(), is(2)); - - DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); - assertThat(hotStats, is(notNullValue())); - assertThat(hotStats.nodeCount, is(3)); - assertThat(hotStats.indexCount, is(1)); - assertThat(hotStats.totalShardCount, is(6)); - assertThat(hotStats.docCount, is(6 * docCount)); - assertThat(hotStats.totalByteCount, is(6 * byteSize)); - assertThat(hotStats.primaryShardCount, is(3)); - assertThat(hotStats.primaryByteCount, is(3 * byteSize)); - assertThat(hotStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size - - DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); - assertThat(warmStats, is(notNullValue())); - assertThat(warmStats.nodeCount, is(0)); - assertThat(warmStats.indexCount, is(1)); - assertThat(warmStats.totalShardCount, is(2)); - assertThat(warmStats.docCount, is(2 * docCount)); - assertThat(warmStats.totalByteCount, is(2 * byteSize)); - assertThat(warmStats.primaryShardCount, is(1)); - assertThat(warmStats.primaryByteCount, is(byteSize)); - assertThat(warmStats.primaryByteCountMedian, is(byteSize)); // All same size - assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size - } - - private static DiscoveryNode newNode(int nodeId, DiscoveryNodeRole... roles) { - return DiscoveryNodeUtils.builder("node_" + nodeId).roles(Set.of(roles)).build(); - } - - private static IndexMetadata indexMetadata(String indexName, int numberOfShards, int numberOfReplicas, String... dataTierPrefs) { - Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), numberOfShards, numberOfReplicas).put( - SETTING_CREATION_DATE, - System.currentTimeMillis() - ); - - if (dataTierPrefs.length > 1) { - StringBuilder tierBuilder = new StringBuilder(dataTierPrefs[0]); - for (int idx = 1; idx < dataTierPrefs.length; idx++) { - tierBuilder.append(',').append(dataTierPrefs[idx]); - } - settingsBuilder.put(DataTier.TIER_PREFERENCE, tierBuilder.toString()); - } else if (dataTierPrefs.length == 1) { - settingsBuilder.put(DataTier.TIER_PREFERENCE, dataTierPrefs[0]); - } - - return IndexMetadata.builder(indexName).settings(settingsBuilder.build()).timestampRange(IndexLongFieldRange.UNKNOWN).build(); - } - - private static void routeTestShardToNodes( - IndexMetadata index, - int shard, - IndexRoutingTable.Builder indexRoutingTableBuilder, - DiscoveryNode... nodes - ) { - ShardId shardId = new ShardId(index.getIndex(), shard); - IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); - boolean primary = true; - for (DiscoveryNode node : nodes) { - indexShardRoutingBuilder.addShard( - TestShardRouting.newShardRouting(shardId, node.getId(), null, primary, ShardRoutingState.STARTED) - ); - primary = false; - } - indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder); - } - - private List buildNodeStats(ClusterState clusterState, long bytesPerShard, long docsPerShard) { - DiscoveryNodes nodes = clusterState.getNodes(); - RoutingNodes routingNodes = clusterState.getRoutingNodes(); - List nodeStatsList = new ArrayList<>(); - for (DiscoveryNode node : nodes) { - RoutingNode routingNode = routingNodes.node(node.getId()); - if (routingNode == null) { - continue; - } - Map> indexStats = new HashMap<>(); - for (ShardRouting shardRouting : routingNode) { - ShardId shardId = shardRouting.shardId(); - ShardStats shardStat = shardStat(bytesPerShard, docsPerShard, shardRouting); - IndexShardStats shardStats = new IndexShardStats(shardId, new ShardStats[] { shardStat }); - indexStats.computeIfAbsent(shardId.getIndex(), k -> new ArrayList<>()).add(shardStats); - } - NodeIndicesStats nodeIndexStats = new NodeIndicesStats(new CommonStats(), Collections.emptyMap(), indexStats, true); - nodeStatsList.add(mockNodeStats(node, nodeIndexStats)); - } - return nodeStatsList; - } - - private static ShardStats shardStat(long byteCount, long docCount, ShardRouting routing) { - StoreStats storeStats = new StoreStats(randomNonNegativeLong(), byteCount, 0L); - DocsStats docsStats = new DocsStats(docCount, 0L, byteCount); - - CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL); - commonStats.getStore().add(storeStats); - commonStats.getDocs().add(docsStats); - - Path fakePath = PathUtils.get("test/dir/" + routing.shardId().getIndex().getUUID() + "/" + routing.shardId().id()); - ShardPath fakeShardPath = new ShardPath(false, fakePath, fakePath, routing.shardId()); - - return new ShardStats(routing, fakeShardPath, commonStats, null, null, null, false, 0); - } - - private static NodeStats mockNodeStats(DiscoveryNode node, NodeIndicesStats indexStats) { - NodeStats stats = mock(NodeStats.class); - when(stats.getNode()).thenReturn(node); - when(stats.getIndices()).thenReturn(indexStats); - return stats; - } -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java new file mode 100644 index 0000000000000..63cc6e4d7914e --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.IndexLongFieldRange; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.test.ESTestCase; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; + +class DataTierUsageFixtures extends ESTestCase { + + private static final CommonStats COMMON_STATS = new CommonStats( + CommonStatsFlags.NONE.set(CommonStatsFlags.Flag.Docs, true).set(CommonStatsFlags.Flag.Store, true) + ); + + static DiscoveryNode newNode(int nodeId, DiscoveryNodeRole... roles) { + return DiscoveryNodeUtils.builder("node_" + nodeId).roles(Set.of(roles)).build(); + } + + static void routeTestShardToNodes( + IndexMetadata index, + int shard, + IndexRoutingTable.Builder indexRoutingTableBuilder, + DiscoveryNode... nodes + ) { + ShardId shardId = new ShardId(index.getIndex(), shard); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); + boolean primary = true; + for (DiscoveryNode node : nodes) { + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(shardId, node.getId(), null, primary, ShardRoutingState.STARTED) + ); + primary = false; + } + indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder); + } + + static NodeIndicesStats buildNodeIndicesStats(RoutingNode routingNode, long bytesPerShard, long docsPerShard) { + Map> indexStats = new HashMap<>(); + for (ShardRouting shardRouting : routingNode) { + ShardId shardId = shardRouting.shardId(); + ShardStats shardStat = shardStat(bytesPerShard, docsPerShard, shardRouting); + IndexShardStats shardStats = new IndexShardStats(shardId, new ShardStats[] { shardStat }); + indexStats.computeIfAbsent(shardId.getIndex(), k -> new ArrayList<>()).add(shardStats); + } + return new NodeIndicesStats(COMMON_STATS, Map.of(), indexStats, true); + } + + private static ShardStats shardStat(long byteCount, long docCount, ShardRouting routing) { + StoreStats storeStats = new StoreStats(randomNonNegativeLong(), byteCount, 0L); + DocsStats docsStats = new DocsStats(docCount, 0L, byteCount); + Path fakePath = PathUtils.get("test/dir/" + routing.shardId().getIndex().getUUID() + "/" + routing.shardId().id()); + ShardPath fakeShardPath = new ShardPath(false, fakePath, fakePath, routing.shardId()); + CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL); + commonStats.getStore().add(storeStats); + commonStats.getDocs().add(docsStats); + return new ShardStats(routing, fakeShardPath, commonStats, null, null, null, false, 0); + } + + static IndexMetadata indexMetadata(String indexName, int numberOfShards, int numberOfReplicas, String... dataTierPrefs) { + Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), numberOfShards, numberOfReplicas).put( + SETTING_CREATION_DATE, + System.currentTimeMillis() + ); + + if (dataTierPrefs.length > 1) { + StringBuilder tierBuilder = new StringBuilder(dataTierPrefs[0]); + for (int idx = 1; idx < dataTierPrefs.length; idx++) { + tierBuilder.append(',').append(dataTierPrefs[idx]); + } + settingsBuilder.put(DataTier.TIER_PREFERENCE, tierBuilder.toString()); + } else if (dataTierPrefs.length == 1) { + settingsBuilder.put(DataTier.TIER_PREFERENCE, dataTierPrefs[0]); + } + + return IndexMetadata.builder(indexName).settings(settingsBuilder.build()).timestampRange(IndexLongFieldRange.UNKNOWN).build(); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsageTests.java similarity index 97% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsageTests.java index e5f37dfb5764c..0951408441b3f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersFeatureSetUsageTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core; +package org.elasticsearch.xpack.core.datatiers; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.io.stream.Writeable; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java new file mode 100644 index 0000000000000..bb8dce7db0e23 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java @@ -0,0 +1,535 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.IntStream; + +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.indexMetadata; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.newNode; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.routeTestShardToNodes; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class DataTiersUsageTransportActionTests extends ESTestCase { + + private long byteSize; + private long docCount; + + @Before + public void setup() { + byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB + docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million + } + + public void testTierIndices() { + DiscoveryNode dataNode = newNode(0, DiscoveryNodeRole.DATA_ROLE); + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + discoBuilder.add(dataNode); + + IndexMetadata hotIndex1 = indexMetadata("hot-1", 1, 0, DataTier.DATA_HOT); + IndexMetadata hotIndex2 = indexMetadata("hot-2", 1, 0, DataTier.DATA_HOT); + IndexMetadata warmIndex1 = indexMetadata("warm-1", 1, 0, DataTier.DATA_WARM); + IndexMetadata coldIndex1 = indexMetadata("cold-1", 1, 0, DataTier.DATA_COLD); + IndexMetadata coldIndex2 = indexMetadata("cold-2", 1, 0, DataTier.DATA_COLD, DataTier.DATA_WARM); // Prefers cold over warm + IndexMetadata nonTiered = indexMetadata("non-tier", 1, 0); // No tier + IndexMetadata hotIndex3 = indexMetadata("hot-3", 1, 0, DataTier.DATA_HOT); + + Metadata.Builder metadataBuilder = Metadata.builder() + .put(hotIndex1, false) + .put(hotIndex2, false) + .put(warmIndex1, false) + .put(coldIndex1, false) + .put(coldIndex2, false) + .put(nonTiered, false) + .put(hotIndex3, false) + .generateClusterUuidIfNeeded(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + routingTableBuilder.add(getIndexRoutingTable(hotIndex1, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(hotIndex2, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(hotIndex2, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(warmIndex1, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(coldIndex1, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(coldIndex2, dataNode)); + routingTableBuilder.add(getIndexRoutingTable(nonTiered, dataNode)); + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .nodes(discoBuilder) + .metadata(metadataBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + Map> result = DataTiersUsageTransportAction.getIndicesGroupedByTier( + clusterState, + List.of(new NodeDataTiersUsage(dataNode, Map.of(DataTier.DATA_WARM, createStats(5, 5, 0, 10)))) + ); + assertThat(result.keySet(), equalTo(Set.of(DataTier.DATA_HOT, DataTier.DATA_WARM, DataTier.DATA_COLD))); + assertThat(result.get(DataTier.DATA_HOT), equalTo(Set.of(hotIndex1.getIndex().getName(), hotIndex2.getIndex().getName()))); + assertThat(result.get(DataTier.DATA_WARM), equalTo(Set.of(warmIndex1.getIndex().getName()))); + assertThat(result.get(DataTier.DATA_COLD), equalTo(Set.of(coldIndex1.getIndex().getName(), coldIndex2.getIndex().getName()))); + } + + public void testCalculateMAD() { + assertThat(DataTiersUsageTransportAction.computeMedianAbsoluteDeviation(TDigestState.create(10)), equalTo(0L)); + + TDigestState sketch = TDigestState.create(randomDoubleBetween(1, 1000, false)); + sketch.add(1); + sketch.add(1); + sketch.add(2); + sketch.add(2); + sketch.add(4); + sketch.add(6); + sketch.add(9); + assertThat(DataTiersUsageTransportAction.computeMedianAbsoluteDeviation(sketch), equalTo(1L)); + } + + public void testCalculateStatsNoTiers() { + // Nodes: 0 Tiered Nodes, 1 Data Node, no indices on tiered nodes + DiscoveryNode leader = newNode(0, DiscoveryNodeRole.MASTER_ROLE); + DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_ROLE); + + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage(dataNode1, Map.of()) + ); + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages, + Map.of() + ); + + // Verify - No results when no tiers present + assertThat(tierSpecificStats.size(), is(0)); + } + + public void testCalculateStatsTieredNodesOnly() { + // Nodes: 1 Data, 1 Hot, 1 Warm, 1 Cold, 1 Frozen + DiscoveryNode leader = newNode(0, DiscoveryNodeRole.MASTER_ROLE); + DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_ROLE); + DiscoveryNode hotNode1 = newNode(2, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); + DiscoveryNode warmNode1 = newNode(3, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + DiscoveryNode coldNode1 = newNode(4, DiscoveryNodeRole.DATA_COLD_NODE_ROLE); + DiscoveryNode frozenNode1 = newNode(5, DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE); + + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage(dataNode1, Map.of()), + new NodeDataTiersUsage(hotNode1, Map.of()), + new NodeDataTiersUsage(warmNode1, Map.of()), + new NodeDataTiersUsage(coldNode1, Map.of()), + new NodeDataTiersUsage(frozenNode1, Map.of()) + ); + + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages, + Map.of() + ); + + // Verify - Results are present, but they lack index numbers because none are tiered + assertThat(tierSpecificStats.size(), is(4)); + + DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); + assertThat(hotStats, is(notNullValue())); + assertThat(hotStats.nodeCount, is(1)); + assertThat(hotStats.indexCount, is(0)); + assertThat(hotStats.totalShardCount, is(0)); + assertThat(hotStats.docCount, is(0L)); + assertThat(hotStats.totalByteCount, is(0L)); + assertThat(hotStats.primaryShardCount, is(0)); + assertThat(hotStats.primaryByteCount, is(0L)); + assertThat(hotStats.primaryByteCountMedian, is(0L)); // All same size + assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size + + DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); + assertThat(warmStats, is(notNullValue())); + assertThat(warmStats.nodeCount, is(1)); + assertThat(warmStats.indexCount, is(0)); + assertThat(warmStats.totalShardCount, is(0)); + assertThat(warmStats.docCount, is(0L)); + assertThat(warmStats.totalByteCount, is(0L)); + assertThat(warmStats.primaryShardCount, is(0)); + assertThat(warmStats.primaryByteCount, is(0L)); + assertThat(warmStats.primaryByteCountMedian, is(0L)); // All same size + assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size + + DataTiersFeatureSetUsage.TierSpecificStats coldStats = tierSpecificStats.get(DataTier.DATA_COLD); + assertThat(coldStats, is(notNullValue())); + assertThat(coldStats.nodeCount, is(1)); + assertThat(coldStats.indexCount, is(0)); + assertThat(coldStats.totalShardCount, is(0)); + assertThat(coldStats.docCount, is(0L)); + assertThat(coldStats.totalByteCount, is(0L)); + assertThat(coldStats.primaryShardCount, is(0)); + assertThat(coldStats.primaryByteCount, is(0L)); + assertThat(coldStats.primaryByteCountMedian, is(0L)); // All same size + assertThat(coldStats.primaryShardBytesMAD, is(0L)); // All same size + + DataTiersFeatureSetUsage.TierSpecificStats frozenStats = tierSpecificStats.get(DataTier.DATA_FROZEN); + assertThat(frozenStats, is(notNullValue())); + assertThat(frozenStats.nodeCount, is(1)); + assertThat(frozenStats.indexCount, is(0)); + assertThat(frozenStats.totalShardCount, is(0)); + assertThat(frozenStats.docCount, is(0L)); + assertThat(frozenStats.totalByteCount, is(0L)); + assertThat(frozenStats.primaryShardCount, is(0)); + assertThat(frozenStats.primaryByteCount, is(0L)); + assertThat(frozenStats.primaryByteCountMedian, is(0L)); // All same size + assertThat(frozenStats.primaryShardBytesMAD, is(0L)); // All same size + } + + public void testCalculateStatsTieredIndicesOnly() { + // Nodes: 3 Data, 0 Tiered - Only hosting indices on generic data nodes + int nodeId = 0; + DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); + DiscoveryNode dataNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); + DiscoveryNode dataNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); + DiscoveryNode dataNode3 = newNode(nodeId, DiscoveryNodeRole.DATA_ROLE); + + // Indices: + // 1 Hot index: 3 primaries, 3 replicas one on each node + // 2 Warm indices, each index 1 primary one replica + // 3 Cold indices, each index 1 primary on a different node + String hotIndex = "hot_index_1"; + String warmIndex1 = "warm_index_1"; + String warmIndex2 = "warm_index_2"; + String coldIndex1 = "cold_index_1"; + String coldIndex2 = "cold_index_2"; + String coldIndex3 = "cold_index_3"; + + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage( + dataNode1, + Map.of( + DataTier.DATA_HOT, + createStats(1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(0, 2, docCount, byteSize), + DataTier.DATA_COLD, + createStats(1, 1, docCount, byteSize) + ) + ), + new NodeDataTiersUsage( + dataNode2, + Map.of( + DataTier.DATA_HOT, + createStats(1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(1, 1, docCount, byteSize), + DataTier.DATA_COLD, + createStats(1, 1, docCount, byteSize) + ) + ), + new NodeDataTiersUsage( + dataNode3, + Map.of( + DataTier.DATA_HOT, + createStats(1, 2, docCount, byteSize), + DataTier.DATA_WARM, + createStats(1, 1, docCount, byteSize), + DataTier.DATA_COLD, + createStats(1, 1, docCount, byteSize) + ) + ) + ); + // Calculate usage + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages, + Map.of( + DataTier.DATA_HOT, + Set.of(hotIndex), + DataTier.DATA_WARM, + Set.of(warmIndex1, warmIndex2), + DataTier.DATA_COLD, + Set.of(coldIndex1, coldIndex2, coldIndex3) + ) + ); + + // Verify - Index stats exist for the tiers, but no tiered nodes are found + assertThat(tierSpecificStats.size(), is(3)); + + DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); + assertThat(hotStats, is(notNullValue())); + assertThat(hotStats.nodeCount, is(0)); + assertThat(hotStats.indexCount, is(1)); + assertThat(hotStats.totalShardCount, is(6)); + assertThat(hotStats.docCount, is(6 * docCount)); + assertThat(hotStats.totalByteCount, is(6 * byteSize)); + assertThat(hotStats.primaryShardCount, is(3)); + assertThat(hotStats.primaryByteCount, is(3 * byteSize)); + assertThat(hotStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size + + DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); + assertThat(warmStats, is(notNullValue())); + assertThat(warmStats.nodeCount, is(0)); + assertThat(warmStats.indexCount, is(2)); + assertThat(warmStats.totalShardCount, is(4)); + assertThat(warmStats.docCount, is(4 * docCount)); + assertThat(warmStats.totalByteCount, is(4 * byteSize)); + assertThat(warmStats.primaryShardCount, is(2)); + assertThat(warmStats.primaryByteCount, is(2 * byteSize)); + assertThat(warmStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size + + DataTiersFeatureSetUsage.TierSpecificStats coldStats = tierSpecificStats.get(DataTier.DATA_COLD); + assertThat(coldStats, is(notNullValue())); + assertThat(coldStats.nodeCount, is(0)); + assertThat(coldStats.indexCount, is(3)); + assertThat(coldStats.totalShardCount, is(3)); + assertThat(coldStats.docCount, is(3 * docCount)); + assertThat(coldStats.totalByteCount, is(3 * byteSize)); + assertThat(coldStats.primaryShardCount, is(3)); + assertThat(coldStats.primaryByteCount, is(3 * byteSize)); + assertThat(coldStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(coldStats.primaryShardBytesMAD, is(0L)); // All same size + } + + public void testCalculateStatsReasonableCase() { + // Nodes: 3 Hot, 5 Warm, 1 Cold + int nodeId = 0; + DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); + DiscoveryNode hotNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); + DiscoveryNode hotNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); + DiscoveryNode hotNode3 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); + DiscoveryNode warmNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + DiscoveryNode warmNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + DiscoveryNode warmNode3 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + DiscoveryNode warmNode4 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + DiscoveryNode warmNode5 = newNode(nodeId++, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + DiscoveryNode coldNode1 = newNode(nodeId, DiscoveryNodeRole.DATA_COLD_NODE_ROLE); + + // Indices: + // 1 Hot index: 3 primaries, 3 replicas one on each node + // 2 Warm indices: each index has 1 primary and 1 replica residing in 4 nodes + // 3 Cold indices: 1 primary each on the cold node + String hotIndex1 = "hot_index_1"; + String warmIndex1 = "warm_index_1"; + String warmIndex2 = "warm_index_2"; + String coldIndex1 = "cold_index_1"; + String coldIndex2 = "cold_index_2"; + String coldIndex3 = "cold_index_3"; + + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage(hotNode1, Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize))), + new NodeDataTiersUsage(hotNode2, Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize))), + new NodeDataTiersUsage(hotNode3, Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize))), + new NodeDataTiersUsage(warmNode1, Map.of(DataTier.DATA_WARM, createStats(1, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode2, Map.of(DataTier.DATA_WARM, createStats(0, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode3, Map.of(DataTier.DATA_WARM, createStats(1, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode4, Map.of(DataTier.DATA_WARM, createStats(0, 1, docCount, byteSize))), + new NodeDataTiersUsage(warmNode5, Map.of()), + new NodeDataTiersUsage(coldNode1, Map.of(DataTier.DATA_COLD, createStats(3, 3, docCount, byteSize))) + + ); + // Calculate usage + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages, + Map.of( + DataTier.DATA_HOT, + Set.of(hotIndex1), + DataTier.DATA_WARM, + Set.of(warmIndex1, warmIndex2), + DataTier.DATA_COLD, + Set.of(coldIndex1, coldIndex2, coldIndex3) + ) + ); + + // Verify - Node and Index stats are both collected + assertThat(tierSpecificStats.size(), is(3)); + + DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); + assertThat(hotStats, is(notNullValue())); + assertThat(hotStats.nodeCount, is(3)); + assertThat(hotStats.indexCount, is(1)); + assertThat(hotStats.totalShardCount, is(6)); + assertThat(hotStats.docCount, is(6 * docCount)); + assertThat(hotStats.totalByteCount, is(6 * byteSize)); + assertThat(hotStats.primaryShardCount, is(3)); + assertThat(hotStats.primaryByteCount, is(3 * byteSize)); + assertThat(hotStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size + + DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); + assertThat(warmStats, is(notNullValue())); + assertThat(warmStats.nodeCount, is(5)); + assertThat(warmStats.indexCount, is(2)); + assertThat(warmStats.totalShardCount, is(4)); + assertThat(warmStats.docCount, is(4 * docCount)); + assertThat(warmStats.totalByteCount, is(4 * byteSize)); + assertThat(warmStats.primaryShardCount, is(2)); + assertThat(warmStats.primaryByteCount, is(2 * byteSize)); + assertThat(warmStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size + + DataTiersFeatureSetUsage.TierSpecificStats coldStats = tierSpecificStats.get(DataTier.DATA_COLD); + assertThat(coldStats, is(notNullValue())); + assertThat(coldStats.nodeCount, is(1)); + assertThat(coldStats.indexCount, is(3)); + assertThat(coldStats.totalShardCount, is(3)); + assertThat(coldStats.docCount, is(3 * docCount)); + assertThat(coldStats.totalByteCount, is(3 * byteSize)); + assertThat(coldStats.primaryShardCount, is(3)); + assertThat(coldStats.primaryByteCount, is(3 * byteSize)); + assertThat(coldStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(coldStats.primaryShardBytesMAD, is(0L)); // All same size + } + + public void testCalculateStatsMixedTiers() { + // Nodes: 3 Hot+Warm - Nodes that are marked as part of multiple tiers + int nodeId = 0; + DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); + + DiscoveryNode mixedNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + DiscoveryNode mixedNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + DiscoveryNode mixedNode3 = newNode(nodeId, DiscoveryNodeRole.DATA_HOT_NODE_ROLE, DiscoveryNodeRole.DATA_WARM_NODE_ROLE); + + String hotIndex1 = "hot_index_1"; + String warmIndex1 = "warm_index_1"; + String warmIndex2 = "warm_index_2"; + + // Indices: 1 Hot index, 2 Warm indices + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage( + mixedNode1, + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(1, 2, docCount, byteSize)) + ), + new NodeDataTiersUsage( + mixedNode2, + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(0, 1, docCount, byteSize)) + ), + new NodeDataTiersUsage( + mixedNode3, + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(1, 1, docCount, byteSize)) + ) + ); + + // Calculate usage + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages, + Map.of(DataTier.DATA_HOT, Set.of(hotIndex1), DataTier.DATA_WARM, Set.of(warmIndex1, warmIndex2)) + ); + + // Verify - Index stats are separated by their preferred tier, instead of counted + // toward multiple tiers based on their current routing. Nodes are counted for each tier they are in. + assertThat(tierSpecificStats.size(), is(2)); + + DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); + assertThat(hotStats, is(notNullValue())); + assertThat(hotStats.nodeCount, is(3)); + assertThat(hotStats.indexCount, is(1)); + assertThat(hotStats.totalShardCount, is(6)); + assertThat(hotStats.docCount, is(6 * docCount)); + assertThat(hotStats.totalByteCount, is(6 * byteSize)); + assertThat(hotStats.primaryShardCount, is(3)); + assertThat(hotStats.primaryByteCount, is(3 * byteSize)); + assertThat(hotStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size + + DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); + assertThat(warmStats, is(notNullValue())); + assertThat(warmStats.nodeCount, is(3)); + assertThat(warmStats.indexCount, is(2)); + assertThat(warmStats.totalShardCount, is(4)); + assertThat(warmStats.docCount, is(4 * docCount)); + assertThat(warmStats.totalByteCount, is(4 * byteSize)); + assertThat(warmStats.primaryShardCount, is(2)); + assertThat(warmStats.primaryByteCount, is(2 * byteSize)); + assertThat(warmStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size + } + + public void testCalculateStatsStuckInWrongTier() { + // Nodes: 3 Hot, 0 Warm - Emulating indices stuck on non-preferred tiers + int nodeId = 0; + DiscoveryNode leader = newNode(nodeId++, DiscoveryNodeRole.MASTER_ROLE); + DiscoveryNode hotNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); + DiscoveryNode hotNode2 = newNode(nodeId++, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); + DiscoveryNode hotNode3 = newNode(nodeId, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); + + String hotIndex1 = "hot_index_1"; + String warmIndex1 = "warm_index_1"; + + List nodeDataTiersUsages = List.of( + new NodeDataTiersUsage(leader, Map.of()), + new NodeDataTiersUsage( + hotNode1, + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(1, 1, docCount, byteSize)) + ), + new NodeDataTiersUsage( + hotNode2, + Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize), DataTier.DATA_WARM, createStats(0, 1, docCount, byteSize)) + ), + new NodeDataTiersUsage(hotNode3, Map.of(DataTier.DATA_HOT, createStats(1, 2, docCount, byteSize))) + ); + + // Calculate usage + Map tierSpecificStats = DataTiersUsageTransportAction.aggregateStats( + nodeDataTiersUsages, + Map.of(DataTier.DATA_HOT, Set.of(hotIndex1), DataTier.DATA_WARM, Set.of(warmIndex1)) + ); + + // Verify - Warm indices are still calculated separately from Hot ones, despite Warm nodes missing + assertThat(tierSpecificStats.size(), is(2)); + + DataTiersFeatureSetUsage.TierSpecificStats hotStats = tierSpecificStats.get(DataTier.DATA_HOT); + assertThat(hotStats, is(notNullValue())); + assertThat(hotStats.nodeCount, is(3)); + assertThat(hotStats.indexCount, is(1)); + assertThat(hotStats.totalShardCount, is(6)); + assertThat(hotStats.docCount, is(6 * docCount)); + assertThat(hotStats.totalByteCount, is(6 * byteSize)); + assertThat(hotStats.primaryShardCount, is(3)); + assertThat(hotStats.primaryByteCount, is(3 * byteSize)); + assertThat(hotStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(hotStats.primaryShardBytesMAD, is(0L)); // All same size + + DataTiersFeatureSetUsage.TierSpecificStats warmStats = tierSpecificStats.get(DataTier.DATA_WARM); + assertThat(warmStats, is(notNullValue())); + assertThat(warmStats.nodeCount, is(0)); + assertThat(warmStats.indexCount, is(1)); + assertThat(warmStats.totalShardCount, is(2)); + assertThat(warmStats.docCount, is(2 * docCount)); + assertThat(warmStats.totalByteCount, is(2 * byteSize)); + assertThat(warmStats.primaryShardCount, is(1)); + assertThat(warmStats.primaryByteCount, is(byteSize)); + assertThat(warmStats.primaryByteCountMedian, is(byteSize)); // All same size + assertThat(warmStats.primaryShardBytesMAD, is(0L)); // All same size + } + + private NodeDataTiersUsage.UsageStats createStats(int primaryShardCount, int totalNumberOfShards, long docCount, long byteSize) { + return new NodeDataTiersUsage.UsageStats( + primaryShardCount > 0 ? IntStream.range(0, primaryShardCount).mapToObj(i -> byteSize).toList() : List.of(), + totalNumberOfShards, + totalNumberOfShards * docCount, + totalNumberOfShards * byteSize + ); + } + + private IndexRoutingTable.Builder getIndexRoutingTable(IndexMetadata indexMetadata, DiscoveryNode node) { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetadata.getIndex()); + routeTestShardToNodes(indexMetadata, 0, indexRoutingTableBuilder, node); + return indexRoutingTableBuilder; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java new file mode 100644 index 0000000000000..fb4291530d037 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportActionTests.java @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datatiers; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.buildNodeIndicesStats; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.indexMetadata; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.newNode; +import static org.elasticsearch.xpack.core.datatiers.DataTierUsageFixtures.routeTestShardToNodes; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class NodesDataTiersUsageTransportActionTests extends ESTestCase { + + private long byteSize; + private long docCount; + + @Before + public void setup() { + byteSize = randomLongBetween(1024L, 1024L * 1024L * 1024L * 30L); // 1 KB to 30 GB + docCount = randomLongBetween(100L, 100000000L); // one hundred to one hundred million + } + + public void testCalculateStatsNoTiers() { + // Nodes: 0 Tiered Nodes, 1 Data Node + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_ROLE); + discoBuilder.add(dataNode1); + discoBuilder.localNodeId(dataNode1.getId()); + + // Indices: 1 Regular index + Metadata.Builder metadataBuilder = Metadata.builder(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + + IndexMetadata index1 = indexMetadata("index_1", 3, 1); + metadataBuilder.put(index1, false).generateClusterUuidIfNeeded(); + + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index1.getIndex()); + routeTestShardToNodes(index1, 0, indexRoutingTableBuilder, dataNode1); + routeTestShardToNodes(index1, 1, indexRoutingTableBuilder, dataNode1); + routeTestShardToNodes(index1, 2, indexRoutingTableBuilder, dataNode1); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + + // Cluster State and create stats responses + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(metadataBuilder) + .nodes(discoBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + NodeIndicesStats nodeIndicesStats = buildNodeIndicesStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + byteSize, + docCount + ); + + // Calculate usage + Map usageStats = NodesDataTiersUsageTransportAction.aggregateStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + clusterState.metadata(), + nodeIndicesStats + ); + + // Verify - No results when no tiers present + assertThat(usageStats.size(), is(0)); + } + + public void testCalculateStatsNoIndices() { + // Nodes: 1 Data, 1 Hot, 1 Warm, 1 Cold, 1 Frozen + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + DiscoveryNode dataNode1 = newNode(1, DiscoveryNodeRole.DATA_HOT_NODE_ROLE); + discoBuilder.add(dataNode1); + discoBuilder.localNodeId(dataNode1.getId()); + + // Indices: 1 Regular index, not hosted on any tiers + Metadata.Builder metadataBuilder = Metadata.builder(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + + // Cluster State and create stats responses + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(metadataBuilder) + .nodes(discoBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + NodeIndicesStats nodeIndicesStats = buildNodeIndicesStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + byteSize, + docCount + ); + + // Calculate usage + Map usageStats = NodesDataTiersUsageTransportAction.aggregateStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + clusterState.metadata(), + nodeIndicesStats + ); + + // Verify - No results when no tiers present + assertThat(usageStats.size(), is(0)); + } + + public void testCalculateStatsTieredIndicesOnly() { + // Nodes: 3 Data, 0 Tiered - Only hosting indices on generic data nodes + int nodeId = 0; + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + + DiscoveryNode dataNode1 = newNode(nodeId++, DiscoveryNodeRole.DATA_ROLE); + discoBuilder.add(dataNode1); + DiscoveryNode dataNode2 = newNode(nodeId, DiscoveryNodeRole.DATA_ROLE); + discoBuilder.add(dataNode2); + + discoBuilder.localNodeId(dataNode1.getId()); + + // Indices: 1 Hot index, 2 Warm indices, 3 Cold indices + Metadata.Builder metadataBuilder = Metadata.builder(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + + IndexMetadata hotIndex1 = indexMetadata("hot_index_1", 3, 1, DataTier.DATA_HOT); + metadataBuilder.put(hotIndex1, false).generateClusterUuidIfNeeded(); + { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(hotIndex1.getIndex()); + routeTestShardToNodes(hotIndex1, 0, indexRoutingTableBuilder, dataNode1, dataNode2); + routeTestShardToNodes(hotIndex1, 1, indexRoutingTableBuilder, dataNode2, dataNode1); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + } + + IndexMetadata warmIndex1 = indexMetadata("warm_index_1", 1, 1, DataTier.DATA_WARM); + metadataBuilder.put(warmIndex1, false).generateClusterUuidIfNeeded(); + { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex1.getIndex()); + routeTestShardToNodes(warmIndex1, 0, indexRoutingTableBuilder, dataNode1, dataNode2); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + } + IndexMetadata warmIndex2 = indexMetadata("warm_index_2", 1, 1, DataTier.DATA_WARM); + metadataBuilder.put(warmIndex2, false).generateClusterUuidIfNeeded(); + { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(warmIndex2.getIndex()); + routeTestShardToNodes(warmIndex2, 0, indexRoutingTableBuilder, dataNode2, dataNode1); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + } + + IndexMetadata coldIndex1 = indexMetadata("cold_index_1", 1, 0, DataTier.DATA_COLD); + metadataBuilder.put(coldIndex1, false).generateClusterUuidIfNeeded(); + { + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(coldIndex1.getIndex()); + routeTestShardToNodes(coldIndex1, 0, indexRoutingTableBuilder, dataNode1); + routingTableBuilder.add(indexRoutingTableBuilder.build()); + } + + // Cluster State and create stats responses + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .nodes(discoBuilder) + .metadata(metadataBuilder) + .routingTable(routingTableBuilder.build()) + .build(); + NodeIndicesStats nodeIndicesStats = buildNodeIndicesStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + byteSize, + docCount + ); + + // Calculate usage + Map usageStats = NodesDataTiersUsageTransportAction.aggregateStats( + clusterState.getRoutingNodes().node(dataNode1.getId()), + clusterState.metadata(), + nodeIndicesStats + ); + + // Verify - Index stats exist for the tiers, but no tiered nodes are found + assertThat(usageStats.size(), is(3)); + + NodeDataTiersUsage.UsageStats hotStats = usageStats.get(DataTier.DATA_HOT); + assertThat(hotStats, is(notNullValue())); + assertThat(hotStats.getPrimaryShardSizes(), equalTo(List.of(byteSize))); + assertThat(hotStats.getTotalShardCount(), is(2)); + assertThat(hotStats.getDocCount(), is(hotStats.getTotalShardCount() * docCount)); + assertThat(hotStats.getTotalSize(), is(hotStats.getTotalShardCount() * byteSize)); + + NodeDataTiersUsage.UsageStats warmStats = usageStats.get(DataTier.DATA_WARM); + assertThat(warmStats, is(notNullValue())); + assertThat(warmStats.getPrimaryShardSizes(), equalTo(List.of(byteSize))); + assertThat(warmStats.getTotalShardCount(), is(2)); + assertThat(warmStats.getDocCount(), is(warmStats.getTotalShardCount() * docCount)); + assertThat(warmStats.getTotalSize(), is(warmStats.getTotalShardCount() * byteSize)); + + NodeDataTiersUsage.UsageStats coldStats = usageStats.get(DataTier.DATA_COLD); + assertThat(coldStats, is(notNullValue())); + assertThat(coldStats.getPrimaryShardSizes(), equalTo(List.of(byteSize))); + assertThat(coldStats.getTotalShardCount(), is(1)); + assertThat(coldStats.getDocCount(), is(coldStats.getTotalShardCount() * docCount)); + assertThat(coldStats.getTotalSize(), is(coldStats.getTotalShardCount() * byteSize)); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 9f490792d800f..fc5f5ba616ab8 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -302,6 +302,7 @@ public class Constants { "cluster:monitor/update/health/info", "cluster:monitor/ingest/geoip/stats", "cluster:monitor/main", + "cluster:monitor/nodes/data_tier_usage", "cluster:monitor/nodes/hot_threads", "cluster:monitor/nodes/info", "cluster:monitor/nodes/stats",