Skip to content

Commit 5b22666

Browse files
nik9000nik9000
authored
Implement top_metrics agg (#51155)
The `top_metrics` agg is kind of like `top_hits` but it only works on doc values so it *should* be faster. At this point it is fairly limited in that it only supports a single, numeric sort and a single, numeric metric. And it only fetches the "very topest" document worth of metric. We plan to support returning a configurable number of top metrics, requesting more than one metric and more than one sort. And, eventually, non-numeric sorts and metrics. The trick is doing those things fairly efficiently. Co-Authored by: Zachary Tong <[email protected]>
1 parent cd50922 commit 5b22666

File tree

60 files changed

+4203
-175
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+4203
-175
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,9 @@
5555
import org.elasticsearch.action.update.UpdateRequest;
5656
import org.elasticsearch.action.update.UpdateResponse;
5757
import org.elasticsearch.client.analytics.ParsedStringStats;
58+
import org.elasticsearch.client.analytics.ParsedTopMetrics;
5859
import org.elasticsearch.client.analytics.StringStatsAggregationBuilder;
60+
import org.elasticsearch.client.analytics.TopMetricsAggregationBuilder;
5961
import org.elasticsearch.client.core.CountRequest;
6062
import org.elasticsearch.client.core.CountResponse;
6163
import org.elasticsearch.client.core.GetSourceRequest;
@@ -1929,6 +1931,7 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
19291931
map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
19301932
map.put(CompositeAggregationBuilder.NAME, (p, c) -> ParsedComposite.fromXContent(p, (String) c));
19311933
map.put(StringStatsAggregationBuilder.NAME, (p, c) -> ParsedStringStats.PARSER.parse(p, (String) c));
1934+
map.put(TopMetricsAggregationBuilder.NAME, (p, c) -> ParsedTopMetrics.PARSER.parse(p, (String) c));
19321935
List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
19331936
.map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
19341937
.collect(Collectors.toList());

‎client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedTopMetrics.java‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.client.analytics;
21+
22+
import org.elasticsearch.common.ParseField;
23+
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
24+
import org.elasticsearch.common.xcontent.ObjectParser;
25+
import org.elasticsearch.common.xcontent.ToXContent;
26+
import org.elasticsearch.common.xcontent.XContentBuilder;
27+
import org.elasticsearch.common.xcontent.XContentParser;
28+
import org.elasticsearch.common.xcontent.XContentParserUtils;
29+
import org.elasticsearch.search.aggregations.ParsedAggregation;
30+
31+
import java.io.IOException;
32+
import java.util.HashMap;
33+
import java.util.List;
34+
import java.util.Map;
35+
36+
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
37+
38+
/**
39+
* Results of the {@code top_metrics} aggregation.
40+
*/
41+
public class ParsedTopMetrics extends ParsedAggregation {
42+
private static final ParseField TOP_FIELD = new ParseField("top");
43+
44+
private final List<TopMetrics> topMetrics;
45+
46+
private ParsedTopMetrics(String name, List<TopMetrics> topMetrics) {
47+
setName(name);
48+
this.topMetrics = topMetrics;
49+
}
50+
51+
/**
52+
* The list of top metrics, in sorted order.
53+
*/
54+
public List<TopMetrics> getTopMetrics() {
55+
return topMetrics;
56+
}
57+
58+
@Override
59+
public String getType() {
60+
return TopMetricsAggregationBuilder.NAME;
61+
}
62+
63+
@Override
64+
protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
65+
builder.startArray(TOP_FIELD.getPreferredName());
66+
for (TopMetrics top : topMetrics) {
67+
top.toXContent(builder, params);
68+
}
69+
return builder.endArray();
70+
}
71+
72+
public static final ConstructingObjectParser<ParsedTopMetrics, String> PARSER = new ConstructingObjectParser<>(
73+
TopMetricsAggregationBuilder.NAME, true, (args, name) -> {
74+
@SuppressWarnings("unchecked")
75+
List<TopMetrics> topMetrics = (List<TopMetrics>) args[0];
76+
return new ParsedTopMetrics(name, topMetrics);
77+
});
78+
static {
79+
PARSER.declareObjectArray(constructorArg(), (p, c) -> TopMetrics.PARSER.parse(p, null), TOP_FIELD);
80+
ParsedAggregation.declareAggregationFields(PARSER);
81+
}
82+
83+
/**
84+
* The metrics belonging to the document with the "top" sort key.
85+
*/
86+
public static class TopMetrics implements ToXContent {
87+
private static final ParseField SORT_FIELD = new ParseField("sort");
88+
private static final ParseField METRICS_FIELD = new ParseField("metrics");
89+
90+
private final List<Object> sort;
91+
private final Map<String, Double> metrics;
92+
93+
private TopMetrics(List<Object> sort, Map<String, Double> metrics) {
94+
this.sort = sort;
95+
this.metrics = metrics;
96+
}
97+
98+
/**
99+
* The sort key for these top metrics.
100+
*/
101+
public List<Object> getSort() {
102+
return sort;
103+
}
104+
105+
/**
106+
* The top metric values returned by the aggregation.
107+
*/
108+
public Map<String, Double> getMetrics() {
109+
return metrics;
110+
}
111+
112+
private static final ConstructingObjectParser<TopMetrics, Void> PARSER = new ConstructingObjectParser<>("top", true,
113+
(args, name) -> {
114+
@SuppressWarnings("unchecked")
115+
List<Object> sort = (List<Object>) args[0];
116+
@SuppressWarnings("unchecked")
117+
Map<String, Double> metrics = (Map<String, Double>) args[1];
118+
return new TopMetrics(sort, metrics);
119+
});
120+
static {
121+
PARSER.declareFieldArray(constructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p),
122+
SORT_FIELD, ObjectParser.ValueType.VALUE_ARRAY);
123+
PARSER.declareObject(constructorArg(), (p, c) -> p.map(HashMap::new, XContentParser::doubleValue), METRICS_FIELD);
124+
}
125+
126+
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
127+
builder.startObject();
128+
builder.field(SORT_FIELD.getPreferredName(), sort);
129+
builder.field(METRICS_FIELD.getPreferredName(), metrics);
130+
builder.endObject();
131+
return builder;
132+
};
133+
}
134+
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/TopMetricsAggregationBuilder.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.client.analytics;
21+
22+
import org.elasticsearch.common.io.stream.StreamOutput;
23+
import org.elasticsearch.common.io.stream.Writeable;
24+
import org.elasticsearch.common.xcontent.XContentBuilder;
25+
import org.elasticsearch.index.query.QueryRewriteContext;
26+
import org.elasticsearch.index.query.QueryShardContext;
27+
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
28+
import org.elasticsearch.search.aggregations.AggregationBuilder;
29+
import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
30+
import org.elasticsearch.search.aggregations.AggregatorFactory;
31+
import org.elasticsearch.search.builder.SearchSourceBuilder;
32+
import org.elasticsearch.search.sort.SortBuilder;
33+
34+
import java.io.IOException;
35+
import java.util.Map;
36+
37+
/**
38+
* Builds the Top Metrics aggregation request.
39+
* <p>
40+
* NOTE: This extends {@linkplain AbstractAggregationBuilder} for compatibility
41+
* with {@link SearchSourceBuilder#aggregation(AggregationBuilder)} but it
42+
* doesn't support any "server" side things like
43+
* {@linkplain Writeable#writeTo(StreamOutput)},
44+
* {@linkplain AggregationBuilder#rewrite(QueryRewriteContext)}, or
45+
* {@linkplain AbstractAggregationBuilder#build(QueryShardContext, AggregatorFactory)}.
46+
*/
47+
public class TopMetricsAggregationBuilder extends AbstractAggregationBuilder<TopMetricsAggregationBuilder> {
48+
public static final String NAME = "top_metrics";
49+
50+
private final SortBuilder<?> sort;
51+
private final String metric;
52+
53+
/**
54+
* Build the request.
55+
* @param name the name of the metric
56+
* @param sort the sort key used to select the top metrics
57+
* @param metric the name of the field to select
58+
*/
59+
public TopMetricsAggregationBuilder(String name, SortBuilder<?> sort, String metric) {
60+
super(name);
61+
this.sort = sort;
62+
this.metric = metric;
63+
}
64+
65+
@Override
66+
public String getType() {
67+
return NAME;
68+
}
69+
70+
@Override
71+
protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
72+
builder.startObject();
73+
{
74+
builder.startArray("sort");
75+
sort.toXContent(builder, params);
76+
builder.endArray();
77+
builder.startObject("metric").field("field", metric).endObject();
78+
}
79+
return builder.endObject();
80+
}
81+
82+
@Override
83+
protected void doWriteTo(StreamOutput out) throws IOException {
84+
throw new UnsupportedOperationException();
85+
}
86+
87+
@Override
88+
protected AggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent, Builder subfactoriesBuilder)
89+
throws IOException {
90+
throw new UnsupportedOperationException();
91+
}
92+
93+
@Override
94+
protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, Object> metaData) {
95+
throw new UnsupportedOperationException();
96+
}
97+
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -676,7 +676,11 @@ public void testDefaultNamedXContents() {
676676
List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getDefaultNamedXContents();
677677
int expectedInternalAggregations = InternalAggregationTestCase.getDefaultNamedXContents().size();
678678
int expectedSuggestions = 3;
679+
680+
// Explicitly check for metrics from the analytics module because they aren't in InternalAggregationTestCase
679681
assertTrue(namedXContents.removeIf(e -> e.name.getPreferredName().equals("string_stats")));
682+
assertTrue(namedXContents.removeIf(e -> e.name.getPreferredName().equals("top_metrics")));
683+
680684
assertEquals(expectedInternalAggregations + expectedSuggestions, namedXContents.size());
681685
Map<Class<?>, Integer> categories = new HashMap<>();
682686
for (NamedXContentRegistry.Entry namedXContent : namedXContents) {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/AnalyticsAggsIT.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,21 @@
2727
import org.elasticsearch.client.ESRestHighLevelClientTestCase;
2828
import org.elasticsearch.client.RequestOptions;
2929
import org.elasticsearch.common.xcontent.XContentType;
30+
import org.elasticsearch.search.sort.FieldSortBuilder;
31+
import org.elasticsearch.search.sort.SortOrder;
3032

3133
import java.io.IOException;
3234

35+
import static java.util.Collections.singletonList;
36+
import static java.util.Collections.singletonMap;
3337
import static org.hamcrest.Matchers.aMapWithSize;
3438
import static org.hamcrest.Matchers.closeTo;
3539
import static org.hamcrest.Matchers.equalTo;
3640
import static org.hamcrest.Matchers.hasEntry;
41+
import static org.hamcrest.Matchers.hasSize;
3742

3843
public class AnalyticsAggsIT extends ESRestHighLevelClientTestCase {
39-
public void testBasic() throws IOException {
44+
public void testStringStats() throws IOException {
4045
BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
4146
bulk.add(new IndexRequest().source(XContentType.JSON, "message", "trying out elasticsearch"));
4247
bulk.add(new IndexRequest().source(XContentType.JSON, "message", "more words"));
@@ -55,4 +60,20 @@ public void testBasic() throws IOException {
5560
assertThat(stats.getDistribution(), hasEntry(equalTo("r"), closeTo(.12, .005)));
5661
assertThat(stats.getDistribution(), hasEntry(equalTo("t"), closeTo(.09, .005)));
5762
}
63+
64+
public void testBasic() throws IOException {
65+
BulkRequest bulk = new BulkRequest("test").setRefreshPolicy(RefreshPolicy.IMMEDIATE);
66+
bulk.add(new IndexRequest().source(XContentType.JSON, "s", 1, "v", 2));
67+
bulk.add(new IndexRequest().source(XContentType.JSON, "s", 2, "v", 3));
68+
highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
69+
SearchRequest search = new SearchRequest("test");
70+
search.source().aggregation(new TopMetricsAggregationBuilder(
71+
"test", new FieldSortBuilder("s").order(SortOrder.DESC), "v"));
72+
SearchResponse response = highLevelClient().search(search, RequestOptions.DEFAULT);
73+
ParsedTopMetrics top = response.getAggregations().get("test");
74+
assertThat(top.getTopMetrics(), hasSize(1));
75+
ParsedTopMetrics.TopMetrics metric = top.getTopMetrics().get(0);
76+
assertThat(metric.getSort(), equalTo(singletonList(2)));
77+
assertThat(metric.getMetrics(), equalTo(singletonMap("v", 3.0)));
78+
}
5879
}

‎docs/java-rest/high-level/aggs-builders.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ This page lists all the available aggregations with their corresponding `Aggrega
2525
| {ref}/search-aggregations-metrics-stats-aggregation.html[Stats] | {agg-ref}/metrics/stats/StatsAggregationBuilder.html[StatsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#stats-java.lang.String-[AggregationBuilders.stats()]
2626
| {ref}/search-aggregations-metrics-sum-aggregation.html[Sum] | {agg-ref}/metrics/sum/SumAggregationBuilder.html[SumAggregationBuilder] | {agg-ref}/AggregationBuilders.html#sum-java.lang.String-[AggregationBuilders.sum()]
2727
| {ref}/search-aggregations-metrics-top-hits-aggregation.html[Top hits] | {agg-ref}/metrics/tophits/TopHitsAggregationBuilder.html[TopHitsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#topHits-java.lang.String-[AggregationBuilders.topHits()]
28+
| {ref}/search-aggregations-metrics-top-metrics.html[Top Metrics] | {javadoc-client}/analytics/TopMetricsAggregationBuilder.html[TopMetricsAggregationBuilder] | None
2829
| {ref}/search-aggregations-metrics-valuecount-aggregation.html[Value Count] | {agg-ref}/metrics/valuecount/ValueCountAggregationBuilder.html[ValueCountAggregationBuilder] | {agg-ref}/AggregationBuilders.html#count-java.lang.String-[AggregationBuilders.count()]
2930
| {ref}/search-aggregations-metrics-string-stats-aggregation.html[String Stats] | {javadoc-client}/analytics/StringStatsAggregationBuilder.html[StringStatsAggregationBuilder] | None
3031
|======

‎docs/reference/aggregations/metrics.asciidoc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ include::metrics/sum-aggregation.asciidoc[]
4141

4242
include::metrics/tophits-aggregation.asciidoc[]
4343

44+
include::metrics/top-metrics-aggregation.asciidoc[]
45+
4446
include::metrics/valuecount-aggregation.asciidoc[]
4547

4648
include::metrics/median-absolute-deviation-aggregation.asciidoc[]

0 commit comments

Comments
 (0)