diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java index 1444f817c8c00..a08b799745437 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java @@ -45,7 +45,7 @@ public static TransformConfig randomTransformConfig() { randomSourceConfig(), randomDestConfig(), randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1000, 1000000)), - randomBoolean() ? null : randomSyncConfig(), + randomBoolean() ? null : randomSyncConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100), randomBoolean() ? null : Instant.now(), diff --git a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java index 2370548c6b61c..ff327d554d1d1 100644 --- a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java +++ b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java @@ -19,11 +19,15 @@ package org.elasticsearch.test.rest; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.Request; +import org.elasticsearch.common.Strings; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -41,14 +45,7 @@ public void testWithRestUsage() throws IOException { Response beforeResponse = client().performRequest(new Request("GET", path)); Map beforeResponseBodyMap = entityAsMap(beforeResponse); assertThat(beforeResponseBodyMap, notNullValue()); - Map before_nodesMap = (Map) beforeResponseBodyMap.get("_nodes"); - assertThat(before_nodesMap, notNullValue()); - Integer beforeTotal = (Integer) before_nodesMap.get("total"); - Integer beforeSuccessful = (Integer) before_nodesMap.get("successful"); - Integer beforeFailed = (Integer) before_nodesMap.get("failed"); - assertThat(beforeTotal, greaterThan(0)); - assertThat(beforeSuccessful, equalTo(beforeTotal)); - assertThat(beforeFailed, equalTo(0)); + int beforeSuccessful = assertSuccess(beforeResponseBodyMap); Map beforeNodesMap = (Map) beforeResponseBodyMap.get("nodes"); assertThat(beforeNodesMap, notNullValue()); @@ -98,14 +95,7 @@ public void testWithRestUsage() throws IOException { Response response = client().performRequest(new Request("GET", "_nodes/usage")); Map responseBodyMap = entityAsMap(response); assertThat(responseBodyMap, notNullValue()); - Map _nodesMap = (Map) responseBodyMap.get("_nodes"); - assertThat(_nodesMap, notNullValue()); - Integer total = (Integer) _nodesMap.get("total"); - Integer successful = (Integer) _nodesMap.get("successful"); - Integer failed = (Integer) _nodesMap.get("failed"); - assertThat(total, greaterThan(0)); - assertThat(successful, equalTo(total)); - assertThat(failed, equalTo(0)); + int successful = assertSuccess(responseBodyMap); Map nodesMap = (Map) responseBodyMap.get("nodes"); assertThat(nodesMap, notNullValue()); @@ -143,4 +133,97 @@ public void testMetricsWithAll() throws IOException { + "\"reason\":\"request [_nodes/usage/_all,rest_actions] contains _all and individual metrics [_all,rest_actions]\"")); } + @SuppressWarnings("unchecked") + public void testAggregationUsage() throws IOException { + // First get the current usage figures + String path = randomFrom("_nodes/usage", "_nodes/usage/aggregations", "_nodes/usage/_all"); + Response beforeResponse = client().performRequest(new Request("GET", path)); + Map beforeResponseBodyMap = entityAsMap(beforeResponse); + assertThat(beforeResponseBodyMap, notNullValue()); + int beforeSuccessful = assertSuccess(beforeResponseBodyMap); + + Map beforeNodesMap = (Map) beforeResponseBodyMap.get("nodes"); + assertThat(beforeNodesMap, notNullValue()); + assertThat(beforeNodesMap.size(), equalTo(beforeSuccessful)); + + Map> beforeCombinedAggsUsage = getTotalUsage(beforeNodesMap); + // Do some requests to get some rest usage stats + Request create = new Request("PUT", "/test"); + create.setJsonEntity("{\"mappings\": {\"properties\": { \"str\": {\"type\": \"keyword\"}, " + + "\"foo\": {\"type\": \"keyword\"}, \"num\": {\"type\": \"long\"}, \"start\": {\"type\": \"date\"} } }}"); + client().performRequest(create); + + Request searchRequest = new Request("GET", "/test/_search"); + SearchSourceBuilder searchSource = new SearchSourceBuilder() + .aggregation(AggregationBuilders.terms("str_terms").field("str.keyword")) + .aggregation(AggregationBuilders.terms("num_terms").field("num")) + .aggregation(AggregationBuilders.avg("num_avg").field("num")); + searchRequest.setJsonEntity(Strings.toString(searchSource)); + searchRequest.setJsonEntity(Strings.toString(searchSource)); + client().performRequest(searchRequest); + + searchRequest = new Request("GET", "/test/_search"); + searchSource = new SearchSourceBuilder() + .aggregation(AggregationBuilders.terms("start").field("start")) + .aggregation(AggregationBuilders.avg("num1").field("num")) + .aggregation(AggregationBuilders.avg("num2").field("num")) + .aggregation(AggregationBuilders.terms("foo").field("foo.keyword")); + String r = Strings.toString(searchSource); + searchRequest.setJsonEntity(Strings.toString(searchSource)); + client().performRequest(searchRequest); + + Response response = client().performRequest(new Request("GET", "_nodes/usage")); + Map responseBodyMap = entityAsMap(response); + assertThat(responseBodyMap, notNullValue()); + int successful = assertSuccess(responseBodyMap); + + Map nodesMap = (Map) responseBodyMap.get("nodes"); + assertThat(nodesMap, notNullValue()); + assertThat(nodesMap.size(), equalTo(successful)); + + Map> afterCombinedAggsUsage = getTotalUsage(nodesMap); + + assertDiff(beforeCombinedAggsUsage, afterCombinedAggsUsage, "terms", "numeric", 1L); + assertDiff(beforeCombinedAggsUsage, afterCombinedAggsUsage, "terms", "date", 1L); + assertDiff(beforeCombinedAggsUsage, afterCombinedAggsUsage, "terms", "bytes", 2L); + assertDiff(beforeCombinedAggsUsage, afterCombinedAggsUsage, "avg", "numeric", 3L); + } + + private void assertDiff(Map> before, Map> after, String agg, String vst, + long diff) { + Long valBefore = before.getOrDefault(agg, Collections.emptyMap()).getOrDefault(vst, 0L); + Long valAfter = after.getOrDefault(agg, Collections.emptyMap()).getOrDefault(vst, 0L); + assertThat(agg + "." + vst, valAfter - valBefore, equalTo(diff) ); + } + + private Map> getTotalUsage(Map nodeUsage) { + Map> combined = new HashMap<>(); + for (Map.Entry nodeEntry : nodeUsage.entrySet()) { + @SuppressWarnings("unchecked") + Map beforeAggsUsage = (Map) ((Map) nodeEntry.getValue()).get("aggregations"); + assertThat(beforeAggsUsage, notNullValue()); + for (Map.Entry aggEntry : beforeAggsUsage.entrySet()) { + @SuppressWarnings("unchecked") Map aggMap = (Map) aggEntry.getValue(); + Map combinedAggMap = combined.computeIfAbsent(aggEntry.getKey(), k -> new HashMap<>()); + for (Map.Entry valSourceEntry : aggMap.entrySet()) { + combinedAggMap.put(valSourceEntry.getKey(), + combinedAggMap.getOrDefault(valSourceEntry.getKey(), 0L) + ((Number) valSourceEntry.getValue()).longValue()); + } + } + } + return combined; + } + + private int assertSuccess(Map responseBodyMap) { + @SuppressWarnings("unchecked") Map nodesResultMap = (Map) responseBodyMap.get("_nodes"); + assertThat(nodesResultMap, notNullValue()); + Integer total = (Integer) nodesResultMap.get("total"); + Integer successful = (Integer) nodesResultMap.get("successful"); + Integer failed = (Integer) nodesResultMap.get("failed"); + assertThat(total, greaterThan(0)); + assertThat(successful, equalTo(total)); + assertThat(failed, equalTo(0)); + return successful; + } + } diff --git a/docs/reference/cluster/nodes-usage.asciidoc b/docs/reference/cluster/nodes-usage.asciidoc index 503568b000698..923e0ae944350 100644 --- a/docs/reference/cluster/nodes-usage.asciidoc +++ b/docs/reference/cluster/nodes-usage.asciidoc @@ -31,15 +31,15 @@ of features for each node. All the nodes selective options are explained ==== {api-path-parms-title} ``:: - (Optional, string) Limits the information returned to the specific metrics. - A comma-separated list of the following options: + (Optional, string) Limits the information returned to the specific metrics. + A comma-separated list of the following options: + -- `_all`:: Returns all stats. - + `rest_actions`:: - Returns the REST actions classname with a count of the number of times + Returns the REST actions classname with a count of the number of times that action has been called on the node. -- @@ -79,11 +79,14 @@ The API returns the following response: "timestamp": 1492553961812, <1> "since": 1492553906606, <2> "rest_actions": { - "org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction": 1, - "org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction": 1, - "org.elasticsearch.rest.action.document.RestGetAction": 1, - "org.elasticsearch.rest.action.search.RestSearchAction": 19, <3> - "org.elasticsearch.rest.action.admin.cluster.RestNodesInfoAction": 36 + "nodes_usage_action": 1, + "create_index_action": 1, + "document_get_action": 1, + "search_action": 19, <3> + "nodes_info_action": 36 + }, + "aggregations": { + ... } } } @@ -94,7 +97,9 @@ The API returns the following response: // TESTRESPONSE[s/1492553961812/$body.$_path/] // TESTRESPONSE[s/1492553906606/$body.$_path/] // TESTRESPONSE[s/"rest_actions": [^}]+}/"rest_actions": $body.$_path/] +// TESTRESPONSE[s/"aggregations": [^}]+}/"aggregations": $body.$_path/] <1> Timestamp for when this nodes usage request was performed. <2> Timestamp for when the usage information recording was started. This is equivalent to the time that the node was started. <3> Search action has been called 19 times for this node. + diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java index 44082b16defb6..f7c1e06ab6899 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java @@ -30,10 +30,13 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin; import java.util.Arrays; import java.util.Collections; +import java.util.List; public class MatrixStatsAggregatorTests extends AggregatorTestCase { @@ -136,4 +139,8 @@ public void testTwoFieldsReduce() throws Exception { } } + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new MatrixAggregationPlugin()); + } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java index 078c1015420bf..349d74daa9d51 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java @@ -36,6 +36,8 @@ import java.io.IOException; import java.util.Map; +import static org.elasticsearch.search.aggregations.support.AggregationUsageService.OTHER_SUBTYPE; + public class ChildrenAggregatorFactory extends ValuesSourceAggregatorFactory { private final Query parentFilter; @@ -84,4 +86,10 @@ protected Aggregator doCreateInternal(ValuesSource rawValuesSource, return asMultiBucketAggregator(this, searchContext, parent); } } + + @Override + public String getStatsSubtype() { + // Child Aggregation is registered in non-standard way, so it might return child's values type + return OTHER_SUBTYPE; + } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java index 8ddc2b1c56bb5..82b332a0ec334 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java @@ -36,6 +36,8 @@ import java.io.IOException; import java.util.Map; +import static org.elasticsearch.search.aggregations.support.AggregationUsageService.OTHER_SUBTYPE; + public class ParentAggregatorFactory extends ValuesSourceAggregatorFactory { private final Query parentFilter; @@ -85,4 +87,10 @@ protected Aggregator doCreateInternal(ValuesSource rawValuesSource, return asMultiBucketAggregator(this, searchContext, children); } } + + @Override + public String getStatsSubtype() { + // Parent Aggregation is registered in non-standard way + return OTHER_SUBTYPE; + } } diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java index 3f2fed591b3d0..7a7bc4e92ed9f 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java @@ -46,8 +46,10 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.join.mapper.MetaJoinFieldMapper; import org.elasticsearch.join.mapper.ParentJoinFieldMapper; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; @@ -328,4 +330,9 @@ private void testCaseTermsParentTerms(Query query, IndexSearcher indexSearcher, LongTerms result = search(indexSearcher, query, aggregationBuilder, fieldType, subFieldType); verify.accept(result); } + + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new ParentJoinPlugin()); + } } diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java index 34ad85aedeac1..cc302955234e5 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java @@ -46,8 +46,10 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.join.mapper.MetaJoinFieldMapper; import org.elasticsearch.join.mapper.ParentJoinFieldMapper; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalMin; import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; @@ -187,4 +189,9 @@ private void testCase(Query query, IndexSearcher indexSearcher, Consumer getSearchPlugins() { + return Collections.singletonList(new ParentJoinPlugin()); + } } diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index fa86389f0db2a..8f97dc1b0d36d 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -67,3 +67,9 @@ - match: { hits.total: 1 } - match: { hits.hits.0._id: q3 } +--- +"Verify nodes usage works": + - do: + nodes.usage: {} + - is_true: nodes + - match: { _nodes.failed: 0 } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodeUsage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodeUsage.java index cd5aeef64313c..41ffef0ff4006 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodeUsage.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodeUsage.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.cluster.node.usage; +import org.elasticsearch.Version; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; @@ -31,15 +32,22 @@ public class NodeUsage extends BaseNodeResponse implements ToXContentFragment { - private long timestamp; - private long sinceTime; - private Map restUsage; + private final long timestamp; + private final long sinceTime; + private final Map restUsage; + private final Map aggregationUsage; + @SuppressWarnings("unchecked") public NodeUsage(StreamInput in) throws IOException { super(in); timestamp = in.readLong(); sinceTime = in.readLong(); restUsage = (Map) in.readGenericValue(); + if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + aggregationUsage = (Map) in.readGenericValue(); + } else { + aggregationUsage = null; + } } /** @@ -54,11 +62,13 @@ public NodeUsage(StreamInput in) throws IOException { * a map containing the counts of the number of times each REST * endpoint has been called */ - public NodeUsage(DiscoveryNode node, long timestamp, long sinceTime, Map restUsage) { + public NodeUsage(DiscoveryNode node, long timestamp, long sinceTime, Map restUsage, + Map aggregationUsage) { super(node); this.timestamp = timestamp; this.sinceTime = sinceTime; this.restUsage = restUsage; + this.aggregationUsage = aggregationUsage; } /** @@ -83,6 +93,14 @@ public Map getRestUsage() { return restUsage; } + /** + * @return a map containing the counts of the number of times each REST + * endpoint has been called + */ + public Map getAggregationUsage() { + return aggregationUsage; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("since", sinceTime); @@ -90,6 +108,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("rest_actions"); builder.map(restUsage); } + if (aggregationUsage != null) { + builder.field("aggregations"); + builder.map(aggregationUsage); + } return builder; } @@ -99,6 +121,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(timestamp); out.writeLong(sinceTime); out.writeGenericValue(restUsage); + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + out.writeGenericValue(aggregationUsage); + } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java index 1a8b7926fe987..10f99e5ab34ad 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.cluster.node.usage; +import org.elasticsearch.Version; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -28,10 +29,14 @@ public class NodesUsageRequest extends BaseNodesRequest { private boolean restActions; + private boolean aggregations; public NodesUsageRequest(StreamInput in) throws IOException { super(in); this.restActions = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + this.aggregations = in.readBoolean(); + } } /** @@ -47,6 +52,7 @@ public NodesUsageRequest(String... nodesIds) { */ public NodesUsageRequest all() { this.restActions = true; + this.aggregations = true; return this; } @@ -73,9 +79,28 @@ public NodesUsageRequest restActions(boolean restActions) { return this; } + + /** + * Should the node rest actions usage statistics be returned. + */ + public boolean aggregations() { + return this.aggregations; + } + + /** + * Should the node rest actions usage statistics be returned. + */ + public NodesUsageRequest aggregations(boolean aggregations) { + this.aggregations = aggregations; + return this; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeBoolean(restActions); + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + out.writeBoolean(aggregations); + } } -} \ No newline at end of file +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java index 8ff1227d6fdfa..a3c6656af6f29 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java @@ -27,24 +27,31 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.aggregations.support.AggregationUsageService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.usage.UsageService; import java.io.IOException; import java.util.List; +import java.util.Map; public class TransportNodesUsageAction extends TransportNodesAction { - private UsageService usageService; + private final UsageService restUsageService; + private final AggregationUsageService aggregationUsageService; + private final long sinceTime; @Inject public TransportNodesUsageAction(ThreadPool threadPool, ClusterService clusterService, TransportService transportService, - ActionFilters actionFilters, UsageService usageService) { + ActionFilters actionFilters, UsageService restUsageService, + AggregationUsageService aggregationUsageService) { super(NodesUsageAction.NAME, threadPool, clusterService, transportService, actionFilters, NodesUsageRequest::new, NodeUsageRequest::new, ThreadPool.Names.MANAGEMENT, NodeUsage.class); - this.usageService = usageService; + this.restUsageService = restUsageService; + this.aggregationUsageService = aggregationUsageService; + this.sinceTime = System.currentTimeMillis(); } @Override @@ -65,7 +72,9 @@ protected NodeUsage newNodeResponse(StreamInput in) throws IOException { @Override protected NodeUsage nodeOperation(NodeUsageRequest nodeUsageRequest) { NodesUsageRequest request = nodeUsageRequest.request; - return usageService.getUsageStats(clusterService.localNode(), request.restActions()); + Map restUsage = request.restActions() ? restUsageService.getRestUsageStats() : null; + Map aggsUsage = request.aggregations() ? aggregationUsageService.getUsageStats() : null; + return new NodeUsage(clusterService.localNode(), System.currentTimeMillis(), sinceTime, restUsage, aggsUsage); } public static class NodeUsageRequest extends BaseNodeRequest { diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index f6e263fb41fd2..7e1900dcb019f 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -58,6 +58,7 @@ import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.support.AggregationUsageService; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.transport.RemoteClusterAware; @@ -116,7 +117,7 @@ public String[] getTypes() { private boolean allowUnmappedFields; private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; - private ValuesSourceRegistry valuesSourceRegistry; + private final ValuesSourceRegistry valuesSourceRegistry; public QueryShardContext(int shardId, IndexSettings indexSettings, @@ -504,4 +505,8 @@ public SimilarityService getSimilarityService() { public BitsetFilterCache getBitsetFilterCache() { return bitsetFilterCache; } + + public AggregationUsageService getUsageService() { + return valuesSourceRegistry.getUsageService(); + } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index c858872e91d47..d0a03893c23a2 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -150,6 +150,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.aggregations.support.AggregationUsageService; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.snapshots.SnapshotShardsService; @@ -581,6 +582,7 @@ protected Node(final Environment initialEnvironment, b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry()); b.bind(IngestService.class).toInstance(ingestService); b.bind(UsageService.class).toInstance(usageService); + b.bind(AggregationUsageService.class).toInstance(searchModule.getValuesSourceRegistry().getUsageService()); b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry); b.bind(MetadataUpgrader.class).toInstance(metadataUpgrader); b.bind(MetaStateService.class).toInstance(metaStateService); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java index 0e8b6370f9a12..1174168078f89 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java @@ -68,6 +68,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } else { nodesUsageRequest.clear(); nodesUsageRequest.restActions(metrics.contains("rest_actions")); + nodesUsageRequest.aggregations(metrics.contains("aggregations")); } return channel -> client.admin().cluster().nodesUsage(nodesUsageRequest, new RestBuilderListener(channel) { diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index c2710c8c711f6..c698068390c62 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -307,14 +307,13 @@ public class SearchModule { private final Settings settings; private final List namedWriteables = new ArrayList<>(); private final List namedXContents = new ArrayList<>(); - private ValuesSourceRegistry valuesSourceRegistry; + private final ValuesSourceRegistry valuesSourceRegistry; /** * Constructs a new SearchModule object * * NOTE: This constructor should not be called in production unless an accurate {@link Settings} object is provided. * When constructed, a static flag is set in Lucene {@link BooleanQuery#setMaxClauseCount} according to the settings. - * * @param settings Current settings * @param transportClient Is this being constructed in the TransportClient or not * @param plugins List of included {@link SearchPlugin} objects. @@ -372,7 +371,8 @@ private ValuesSourceRegistry registerAggregations(List plugins) { .addResultReader(InternalAvg::new) .setAggregatorRegistrar(AvgAggregationBuilder::registerAggregators), builder); registerAggregation(new AggregationSpec(WeightedAvgAggregationBuilder.NAME, WeightedAvgAggregationBuilder::new, - WeightedAvgAggregationBuilder.PARSER).addResultReader(InternalWeightedAvg::new), builder); + WeightedAvgAggregationBuilder.PARSER).addResultReader(InternalWeightedAvg::new) + .setAggregatorRegistrar(WeightedAvgAggregationBuilder::registerUsage), builder); registerAggregation(new AggregationSpec(SumAggregationBuilder.NAME, SumAggregationBuilder::new, SumAggregationBuilder.PARSER) .addResultReader(InternalSum::new) .setAggregatorRegistrar(SumAggregationBuilder::registerAggregators), builder); @@ -529,6 +529,10 @@ private void registerAggregation(AggregationSpec spec, ValuesSourceRegistry.Buil Consumer register = spec.getAggregatorRegistrar(); if (register != null) { register.accept(builder); + } else { + // Register is typically handling usage registration, but for the older aggregations that don't use register, we + // have to register usage explicitly here. + builder.registerUsage(spec.getName().getPreferredName()); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java index 9f39c19d4ab48..85fb368e91cbd 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java @@ -137,6 +137,7 @@ public final String getWriteableName() { @Override public final AggregatorFactory build(QueryShardContext queryShardContext, AggregatorFactory parent) throws IOException { AggregatorFactory factory = doBuild(queryShardContext, parent, factoriesBuilder); + queryShardContext.getUsageService().incAggregationUsage(getType(), factory.getStatsSubtype()); return factory; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java index b308de0327e52..a3546c431077c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java @@ -32,6 +32,8 @@ import java.io.IOException; import java.util.Map; +import static org.elasticsearch.search.aggregations.support.AggregationUsageService.OTHER_SUBTYPE; + public abstract class AggregatorFactory { public static final class MultiBucketAggregatorWrapper extends Aggregator { @@ -238,4 +240,13 @@ protected static Aggregator asMultiBucketAggregator(final AggregatorFactory fact return new MultiBucketAggregatorWrapper(bigArrays, searchContext, parent, factory, first); } + /** + * Returns the aggregation subtype for nodes usage stats. + *

+ * It should match the types registered by calling {@linkplain org.elasticsearch.search.aggregations.support.AggregationUsageService}. + * In other words, it should be ValueSourcesType for the VST aggregations OTHER_SUBTYPE for all other aggregations. + */ + public String getStatsSubtype() { + return OTHER_SUBTYPE; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java index 329d47d1d834b..809fb8fd57d2a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java @@ -42,6 +42,8 @@ import java.io.IOException; import java.util.Map; +import static org.elasticsearch.search.aggregations.support.AggregationUsageService.OTHER_SUBTYPE; + public class GeoDistanceRangeAggregatorFactory extends ValuesSourceAggregatorFactory { @@ -128,4 +130,9 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext ctx) { } + @Override + public String getStatsSubtype() { + // GeoDistanceRangeAggregatorFactory doesn't register itself with ValuesSourceRegistry + return OTHER_SUBTYPE; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java index 5fef17ebd857d..88eea6733165f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java @@ -36,6 +36,8 @@ import java.io.IOException; import java.util.Map; +import static org.elasticsearch.search.aggregations.support.AggregationUsageService.OTHER_SUBTYPE; + public class DiversifiedAggregatorFactory extends ValuesSourceAggregatorFactory { private final int shardSize; @@ -98,4 +100,9 @@ public InternalAggregation buildEmptyAggregation() { }; } + @Override + public String getStatsSubtype() { + // DiversifiedAggregatorFactory doesn't register itself with ValuesSourceRegistry + return OTHER_SUBTYPE; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java index f5183bebc04cc..cdd52115e88a3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java @@ -38,6 +38,7 @@ import org.elasticsearch.search.aggregations.support.MultiValuesSourceParseHelper; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; @@ -57,6 +58,10 @@ public class WeightedAvgAggregationBuilder extends MultiValuesSourceAggregationB MultiValuesSourceParseHelper.declareField(WEIGHT_FIELD.getPreferredName(), PARSER, true, false, false); } + public static void registerUsage(ValuesSourceRegistry.Builder builder) { + builder.registerUsage(NAME, CoreValuesSourceType.NUMERIC); + } + public WeightedAvgAggregationBuilder(String name) { super(name); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java index f962ce8578877..64d620e1f5f8e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java @@ -32,6 +32,8 @@ import java.io.IOException; import java.util.Map; +import static org.elasticsearch.search.aggregations.metrics.WeightedAvgAggregationBuilder.VALUE_FIELD; + class WeightedAvgAggregatorFactory extends MultiValuesSourceAggregatorFactory { WeightedAvgAggregatorFactory(String name, Map configs, @@ -62,4 +64,9 @@ protected Aggregator doCreateInternal(SearchContext searchContext, } return new WeightedAvgAggregator(name, numericMultiVS, format, searchContext, parent, metadata); } + + @Override + public String getStatsSubtype() { + return configs.get(VALUE_FIELD.getPreferredName()).valueSourceType().typeName(); + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationUsageService.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationUsageService.java new file mode 100644 index 0000000000000..e0486208942d4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationUsageService.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.support; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.LongAdder; + +public class AggregationUsageService { + private final Map> aggs; + + public static final String OTHER_SUBTYPE = "other"; + + public static class Builder { + private final Map> aggs; + + public Builder() { + aggs = new HashMap<>(); + } + + public void registerAggregationUsage(String aggregationName) { + registerAggregationUsage(aggregationName, OTHER_SUBTYPE); + } + + public void registerAggregationUsage(String aggregationName, String valuesSourceType) { + Map subAgg = aggs.computeIfAbsent(aggregationName, k -> new HashMap<>()); + if (subAgg.put(valuesSourceType, new LongAdder()) != null) { + throw new IllegalArgumentException("stats for aggregation [" + aggregationName + "][" + valuesSourceType + + "] already registered"); + } + } + + public AggregationUsageService build() { + return new AggregationUsageService(this); + } + } + + private AggregationUsageService(Builder builder) { + this.aggs = builder.aggs; + } + + public void incAggregationUsage(String aggregationName, String valuesSourceType) { + Map valuesSourceMap = aggs.get(aggregationName); + // Not all aggs register their usage at the moment we also don't register them in test context + if (valuesSourceMap != null) { + LongAdder adder = valuesSourceMap.get(valuesSourceType); + if (adder != null) { + adder.increment(); + } + assert adder != null : "Unknown subtype [" + aggregationName + "][" + valuesSourceType + "]"; + } + assert valuesSourceMap != null : "Unknown aggregation [" + aggregationName + "][" + valuesSourceType + "]"; + } + + public Map getUsageStats() { + Map aggsUsageMap = new HashMap<>(); + aggs.forEach((name, agg) -> { + Map aggUsageMap = new HashMap<>(); + agg.forEach((k, v) -> { + long val = v.longValue(); + if (val > 0) { + aggUsageMap.put(k, val); + } + }); + if (aggUsageMap.isEmpty() == false) { + aggsUsageMap.put(name, aggUsageMap); + } + }); + return aggsUsageMap; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index 16b07efa7bfd3..1b275b2451de6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -288,6 +288,11 @@ public String value() { return name().toLowerCase(Locale.ROOT); } + @Override + public String typeName() { + return value(); + } + /** List containing all members of the enumeration. */ public static List ALL_CORE = Arrays.asList(CoreValuesSourceType.values()); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java index 6d03d6f056b69..cfbea3010e6f6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java @@ -59,4 +59,8 @@ protected abstract Aggregator doCreateInternal(ValuesSource valuesSource, boolean collectsFromSingleBucket, Map metadata) throws IOException; + @Override + public String getStatsSubtype() { + return config.valueSourceType().typeName(); + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java index c3a58e4d3b7b1..7605100852b52 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java @@ -41,7 +41,14 @@ * */ public class ValuesSourceRegistry { + public static class Builder { + private final AggregationUsageService.Builder usageServiceBuilder; + + public Builder() { + this.usageServiceBuilder = new AggregationUsageService.Builder(); + } + private final Map>> aggregatorRegistry = new HashMap<>(); /** @@ -59,6 +66,7 @@ public synchronized void register(String aggregationName, ValuesSourceType value aggregatorRegistry.put(aggregationName, new ArrayList<>()); } aggregatorRegistry.get(aggregationName).add(new AbstractMap.SimpleEntry<>(valuesSourceType, aggregatorSupplier)); + registerUsage(aggregationName, valuesSourceType); } /** @@ -76,18 +84,29 @@ public void register(String aggregationName, List valuesSource } } + public void registerUsage(String aggregationName, ValuesSourceType valuesSourceType) { + usageServiceBuilder.registerAggregationUsage(aggregationName, valuesSourceType.typeName()); + } + + public void registerUsage(String aggregationName) { + usageServiceBuilder.registerAggregationUsage(aggregationName); + } + public ValuesSourceRegistry build() { - return new ValuesSourceRegistry(aggregatorRegistry); + return new ValuesSourceRegistry(aggregatorRegistry, usageServiceBuilder.build()); } } /** Maps Aggregation names to (ValuesSourceType, Supplier) pairs, keyed by ValuesSourceType */ + private final AggregationUsageService usageService; private Map>> aggregatorRegistry; - public ValuesSourceRegistry(Map>> aggregatorRegistry) { + public ValuesSourceRegistry(Map>> aggregatorRegistry, + AggregationUsageService usageService) { Map>> tmp = new HashMap<>(); aggregatorRegistry.forEach((key, value) -> tmp.put(key, Collections.unmodifiableList(value))); this.aggregatorRegistry = Collections.unmodifiableMap(tmp); + this.usageService = usageService; } private AggregatorSupplier findMatchingSuppier(ValuesSourceType valuesSourceType, @@ -114,7 +133,7 @@ public AggregatorSupplier getAggregator(ValuesSourceType valuesSourceType, Strin public ValuesSourceType getValuesSourceType(MappedFieldType fieldType, String aggregationName, // TODO: the following arguments are only needed for the legacy case - IndexFieldData indexFieldData, + IndexFieldData indexFieldData, ValueType valueType, ValuesSourceType defaultValuesSourceType) { if (aggregationName != null && aggregatorRegistry.containsKey(aggregationName)) { @@ -144,4 +163,8 @@ && findMatchingSuppier(valuesSourceType, aggregatorRegistry.get(aggregationName) } } } + + public AggregationUsageService getUsageService() { + return usageService; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java index c44bbf02c2070..c10fe9fefd63a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java @@ -99,4 +99,10 @@ ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocVal default DocValueFormat getFormatter(String format, ZoneId tz) { return DocValueFormat.RAW; } + + /** + * Returns the name of the Values Source Type for stats purposes + * @return the name of the Values Source Type + */ + String typeName(); } diff --git a/server/src/main/java/org/elasticsearch/usage/UsageService.java b/server/src/main/java/org/elasticsearch/usage/UsageService.java index df1f53bcbb7c8..a0a55a8b93406 100644 --- a/server/src/main/java/org/elasticsearch/usage/UsageService.java +++ b/server/src/main/java/org/elasticsearch/usage/UsageService.java @@ -39,7 +39,6 @@ package org.elasticsearch.usage; import org.elasticsearch.action.admin.cluster.node.usage.NodeUsage; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.rest.BaseRestHandler; import java.util.HashMap; @@ -53,18 +52,15 @@ public class UsageService { private final Map handlers; - private final long sinceTime; public UsageService() { this.handlers = new HashMap<>(); - this.sinceTime = System.currentTimeMillis(); } /** * Add a REST handler to this service. * - * @param handler - * the {@link BaseRestHandler} to add to the usage service. + * @param handler the {@link BaseRestHandler} to add to the usage service. */ public void addRestHandler(BaseRestHandler handler) { Objects.requireNonNull(handler); @@ -92,28 +88,19 @@ public void addRestHandler(BaseRestHandler handler) { /** * Get the current usage statistics for this node. * - * @param localNode - * the {@link DiscoveryNode} for this node - * @param restActions - * whether to include rest action usage in the returned - * statistics * @return the {@link NodeUsage} representing the usage statistics for this - * node + * node */ - public NodeUsage getUsageStats(DiscoveryNode localNode, boolean restActions) { + public Map getRestUsageStats() { Map restUsageMap; - if (restActions) { - restUsageMap = new HashMap<>(); - handlers.values().forEach(handler -> { - long usageCount = handler.getUsageCount(); - if (usageCount > 0) { - restUsageMap.put(handler.getName(), usageCount); - } - }); - } else { - restUsageMap = null; - } - return new NodeUsage(localNode, System.currentTimeMillis(), sinceTime, restUsageMap); + restUsageMap = new HashMap<>(); + handlers.values().forEach(handler -> { + long usageCount = handler.getUsageCount(); + if (usageCount > 0) { + restUsageMap.put(handler.getName(), usageCount); + } + }); + return restUsageMap; } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index 56206f268e1c0..7f79a267f797c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -39,6 +39,8 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.support.AggregationUsageService; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.junit.BeforeClass; import java.io.IOException; @@ -50,6 +52,8 @@ import java.util.function.Function; import static java.util.Collections.singleton; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ScriptedMetricAggregatorTests extends AggregatorTestCase { @@ -426,8 +430,12 @@ protected QueryShardContext queryShardContextMock(IndexSearcher searcher, MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + ValuesSourceRegistry valuesSourceRegistry = mock(ValuesSourceRegistry.class); + AggregationUsageService.Builder builder = new AggregationUsageService.Builder(); + builder.registerAggregationUsage(ScriptedMetricAggregationBuilder.NAME); + when(valuesSourceRegistry.getUsageService()).thenReturn(builder.build()); return new QueryShardContext(0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, scriptService, xContentRegistry(), writableRegistry(), - null, null, System::currentTimeMillis, null, null, () -> true, null); + null, null, System::currentTimeMillis, null, null, () -> true, valuesSourceRegistry); } } diff --git a/server/src/test/java/org/elasticsearch/usage/UsageServiceTests.java b/server/src/test/java/org/elasticsearch/usage/UsageServiceTests.java index 7c9ac5568aa1c..947a64b3d34e2 100644 --- a/server/src/test/java/org/elasticsearch/usage/UsageServiceTests.java +++ b/server/src/test/java/org/elasticsearch/usage/UsageServiceTests.java @@ -19,26 +19,21 @@ package org.elasticsearch.usage; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.node.usage.NodeUsage; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.aggregations.support.AggregationUsageService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; -import java.net.InetAddress; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; +import static org.elasticsearch.search.aggregations.support.AggregationUsageService.OTHER_SUBTYPE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.sameInstance; public class UsageServiceTests extends ESTestCase { @@ -95,8 +90,6 @@ public void testHandlersWithConflictingNamesButDifferentInstances() { } public void testRestUsage() throws Exception { - DiscoveryNode discoveryNode = new DiscoveryNode("foo", new TransportAddress(InetAddress.getByName("localhost"), 12345), - Version.CURRENT); RestRequest restRequest = new FakeRestRequest(); BaseRestHandler handlerA = new MockRestHandler("a"); BaseRestHandler handlerB = new MockRestHandler("b"); @@ -125,9 +118,7 @@ public void testRestUsage() throws Exception { handlerF.handleRequest(restRequest, null, null); handlerC.handleRequest(restRequest, null, null); handlerD.handleRequest(restRequest, null, null); - NodeUsage usage = usageService.getUsageStats(discoveryNode, true); - assertThat(usage.getNode(), sameInstance(discoveryNode)); - Map restUsage = usage.getRestUsage(); + Map restUsage = usageService.getRestUsageStats(); assertThat(restUsage, notNullValue()); assertThat(restUsage.size(), equalTo(6)); assertThat(restUsage.get("a"), equalTo(4L)); @@ -136,10 +127,48 @@ public void testRestUsage() throws Exception { assertThat(restUsage.get("d"), equalTo(2L)); assertThat(restUsage.get("e"), equalTo(1L)); assertThat(restUsage.get("f"), equalTo(1L)); + } + + @SuppressWarnings("unchecked") + public void testAggsUsage() throws Exception { + AggregationUsageService.Builder builder = new AggregationUsageService.Builder(); + + builder.registerAggregationUsage("a", "x"); + builder.registerAggregationUsage("a", "y"); + builder.registerAggregationUsage("b", "x"); + builder.registerAggregationUsage("c"); + builder.registerAggregationUsage("b", "y"); + builder.registerAggregationUsage("a", "z"); + + AggregationUsageService usageService = builder.build(); + + usageService.incAggregationUsage("a", "x"); + for (int i = 0; i < 2; i++) { + usageService.incAggregationUsage("a", "y"); + } + for (int i = 0; i < 3; i++) { + usageService.incAggregationUsage("a", "z"); + } + for (int i = 0; i < 4; i++) { + usageService.incAggregationUsage("b", "x"); + } + for (int i = 0; i < 5; i++) { + usageService.incAggregationUsage("b", "y"); + } + for (int i = 0; i < 6; i++) { + usageService.incAggregationUsage("c", OTHER_SUBTYPE); + } + - usage = usageService.getUsageStats(discoveryNode, false); - assertThat(usage.getNode(), sameInstance(discoveryNode)); - assertThat(usage.getRestUsage(), nullValue()); + Map aggsUsage = usageService.getUsageStats(); + assertThat(aggsUsage, notNullValue()); + assertThat(aggsUsage.size(), equalTo(3)); + assertThat(((Map) aggsUsage.get("a")).get("x"), equalTo(1L)); + assertThat(((Map) aggsUsage.get("a")).get("y"), equalTo(2L)); + assertThat(((Map) aggsUsage.get("a")).get("z"), equalTo(3L)); + assertThat(((Map) aggsUsage.get("b")).get("x"), equalTo(4L)); + assertThat(((Map) aggsUsage.get("b")).get("y"), equalTo(5L)); + assertThat(((Map) aggsUsage.get("c")).get(OTHER_SUBTYPE), equalTo(6L)); } private class MockRestHandler extends BaseRestHandler { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java index 47139062aff7d..2f39cbb9481c4 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java @@ -109,6 +109,7 @@ public List getAggregations() { TTestAggregationBuilder::new, usage.track(AnalyticsStatsAction.Item.T_TEST, checkLicense(TTestAggregationBuilder.PARSER))) .addResultReader(InternalTTest::new) + .setAggregatorRegistrar(TTestAggregationBuilder::registerUsage) ); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java index 8ff07fda257b5..fbc7d530a844e 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java @@ -56,4 +56,9 @@ public static ValuesSourceType fromString(String name) { public String value() { return name().toLowerCase(Locale.ROOT); } + + @Override + public String typeName() { + return value(); + } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java index 31bc1b9fc0400..a51e766d09ce7 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java @@ -26,6 +26,7 @@ import org.elasticsearch.search.aggregations.support.MultiValuesSourceParseHelper; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; @@ -54,6 +55,10 @@ public class TTestAggregationBuilder extends MultiValuesSourceAggregationBuilder private int tails = 2; + public static void registerUsage(ValuesSourceRegistry.Builder builder) { + builder.registerUsage(NAME, CoreValuesSourceType.NUMERIC); + } + public TTestAggregationBuilder(String name) { super(name); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorFactory.java index c960b76a8135e..5247554a9c2bf 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorFactory.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorFactory.java @@ -26,6 +26,8 @@ import java.io.IOException; import java.util.Map; +import static org.elasticsearch.xpack.analytics.ttest.TTestAggregationBuilder.A_FIELD; + class TTestAggregatorFactory extends MultiValuesSourceAggregatorFactory { private final TTestType testType; @@ -118,4 +120,9 @@ public Weight getWeight(Query filter) { } return null; } + + @Override + public String getStatsSubtype() { + return configs.get(A_FIELD.getPreferredName()).valueSourceType().typeName(); + } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index aa77d5c6418e2..d8eac5e4d353a 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptEngine; @@ -64,9 +65,11 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.sort.SortValue; +import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -377,7 +380,7 @@ public long addWithoutBreaking(long bytes) { * BigArrays allocates the new array before freeing the old one. * That causes us to trip when we're about 2/3 of the way to the * limit. And 2/3 of 190 is 126. Which is pretty much what we - * expect. Sort of. + * expect. Sort of. */ int bucketThatBreaks = 646; for (int b = 0; b < bucketThatBreaks; b++) { @@ -584,4 +587,9 @@ protected ScriptService getMockScriptService() { Map engines = singletonMap(scriptEngine.getType(), scriptEngine); return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); } + + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new AnalyticsPlugin(Settings.EMPTY)); + } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java index f91d2d7673695..5980146ed788e 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptEngine; @@ -35,10 +36,12 @@ import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig; import org.elasticsearch.search.lookup.LeafDocLookup; +import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; @@ -651,4 +654,9 @@ private void testCase(Query query, TTestType type, } testCase(aggregationBuilder, query, buildIndex, verify, fieldType1, fieldType2); } + + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new AnalyticsPlugin(Settings.EMPTY)); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java index 5a077e3e70931..a1e2ee4fb1dea 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java @@ -54,4 +54,4 @@ protected NamedXContentRegistry xContentRegistry() { return namedXContentRegistry; } -} \ No newline at end of file +} diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java index 8d98cc2adbebd..b88664c134a77 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java @@ -44,7 +44,7 @@ public void testCheckDataFeedQuery() { qs.put("use_dis_max", true); Map query = Collections.singletonMap("query_string", qs); deprecatedDatafeed.setQuery(query); - + DeprecationIssue issue = MlDeprecationChecks.checkDataFeedQuery(deprecatedDatafeed.build()); assertNotNull(issue); assertThat(issue.getDetails(), equalTo("[Deprecated field [use_dis_max] used, replaced by [Set [tie_breaker] to 1 instead]]")); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/GeoShapeValuesSourceType.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/GeoShapeValuesSourceType.java index 5704b20c867ae..73f00f5660eeb 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/GeoShapeValuesSourceType.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/GeoShapeValuesSourceType.java @@ -115,6 +115,11 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc }; } + @Override + public String typeName() { + return "geoshape"; + } + @Override public void writeTo(StreamOutput out) throws IOException { diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java index 3d5ac303ed772..fb04f94e213ec 100644 --- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java +++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java @@ -65,7 +65,7 @@ protected SqlQueryRequest createTestInstance() { randomBoolean(), randomBoolean() ); } - + @Override protected Writeable.Reader instanceReader() { return SqlQueryRequest::new; @@ -92,7 +92,7 @@ protected SqlQueryRequest mutateInstance(SqlQueryRequest instance) { mutator.accept(newRequest); return newRequest; } - + private AbstractSqlQueryRequest mutateRequestInfo(SqlQueryRequest oldRequest, SqlQueryRequest newRequest) { RequestInfo requestInfo = randomValueOtherThan(newRequest.requestInfo(), this::randomRequestInfo); newRequest.requestInfo(requestInfo); @@ -106,10 +106,10 @@ private AbstractSqlQueryRequest mutateRequestInfo(SqlQueryRequest oldRequest, Sq param.hasExplicitType(true); } } - + return newRequest; } - + public void testFromXContent() throws IOException { xContentTester(this::createParser, this::createTestInstance, SqlQueryRequestTests::toXContent, this::doParseInstance) .numberOfTestRuns(NUMBER_OF_TEST_RUNS) @@ -126,11 +126,11 @@ public void testTimeZoneNullException() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> sqlQueryRequest.zoneId(null)); assertEquals("time zone may not be null.", e.getMessage()); } - + private RequestInfo randomRequestInfo() { return new RequestInfo(randomFrom(Mode.values()), randomFrom(randomFrom(CLIENT_IDS), requestInfo.clientId())); } - + private TimeValue randomTV() { return TimeValue.parseTimeValue(randomTimeValue(), null, "test"); }