Skip to content

Commit ad3c042

Browse files
tlrxtlrx
authored
[Test] Add unit tests for HDR/TDigest PercentilesAggregators (#24245)
Relates to #22278
1 parent 9d8254f commit ad3c042

File tree

2 files changed

+299
-0
lines changed

2 files changed

+299
-0
lines changed

‎core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregatorTests.java‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
21+
22+
import org.apache.lucene.document.LongPoint;
23+
import org.apache.lucene.document.NumericDocValuesField;
24+
import org.apache.lucene.document.SortedNumericDocValuesField;
25+
import org.apache.lucene.index.DirectoryReader;
26+
import org.apache.lucene.index.IndexReader;
27+
import org.apache.lucene.index.RandomIndexWriter;
28+
import org.apache.lucene.search.FieldValueQuery;
29+
import org.apache.lucene.search.IndexSearcher;
30+
import org.apache.lucene.search.MatchAllDocsQuery;
31+
import org.apache.lucene.search.Query;
32+
import org.apache.lucene.store.Directory;
33+
import org.elasticsearch.common.CheckedConsumer;
34+
import org.elasticsearch.index.mapper.MappedFieldType;
35+
import org.elasticsearch.index.mapper.NumberFieldMapper;
36+
import org.elasticsearch.search.aggregations.AggregatorTestCase;
37+
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder;
38+
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod;
39+
40+
import java.io.IOException;
41+
import java.util.function.Consumer;
42+
43+
import static java.util.Arrays.asList;
44+
import static java.util.Collections.singleton;
45+
46+
public class HDRPercentilesAggregatorTests extends AggregatorTestCase {
47+
48+
public void testNoDocs() throws IOException {
49+
testCase(new MatchAllDocsQuery(), iw -> {
50+
// Intentionally not writing any docs
51+
}, hdr -> {
52+
assertEquals(0L, hdr.state.getTotalCount());
53+
});
54+
}
55+
56+
public void testNoMatchingField() throws IOException {
57+
testCase(new MatchAllDocsQuery(), iw -> {
58+
iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
59+
iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 1)));
60+
}, hdr -> {
61+
assertEquals(0L, hdr.state.getTotalCount());
62+
});
63+
}
64+
65+
public void testSomeMatchesSortedNumericDocValues() throws IOException {
66+
testCase(new FieldValueQuery("number"), iw -> {
67+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 60)));
68+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 40)));
69+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 20)));
70+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 10)));
71+
}, hdr -> {
72+
assertEquals(4L, hdr.state.getTotalCount());
73+
double approximation = 0.05d;
74+
assertEquals(10.0d, hdr.percentile(25), approximation);
75+
assertEquals(20.0d, hdr.percentile(50), approximation);
76+
assertEquals(40.0d, hdr.percentile(75), approximation);
77+
assertEquals(60.0d, hdr.percentile(99), approximation);
78+
});
79+
}
80+
81+
public void testSomeMatchesNumericDocValues() throws IOException {
82+
testCase(new FieldValueQuery("number"), iw -> {
83+
iw.addDocument(singleton(new NumericDocValuesField("number", 60)));
84+
iw.addDocument(singleton(new NumericDocValuesField("number", 40)));
85+
iw.addDocument(singleton(new NumericDocValuesField("number", 20)));
86+
iw.addDocument(singleton(new NumericDocValuesField("number", 10)));
87+
}, hdr -> {
88+
assertEquals(4L, hdr.state.getTotalCount());
89+
double approximation = 0.05d;
90+
assertEquals(10.0d, hdr.percentile(25), approximation);
91+
assertEquals(20.0d, hdr.percentile(50), approximation);
92+
assertEquals(40.0d, hdr.percentile(75), approximation);
93+
assertEquals(60.0d, hdr.percentile(99), approximation);
94+
});
95+
}
96+
97+
public void testQueryFiltering() throws IOException {
98+
final CheckedConsumer<RandomIndexWriter, IOException> docs = iw -> {
99+
iw.addDocument(asList(new LongPoint("row", 4), new SortedNumericDocValuesField("number", 60)));
100+
iw.addDocument(asList(new LongPoint("row", 3), new SortedNumericDocValuesField("number", 40)));
101+
iw.addDocument(asList(new LongPoint("row", 2), new SortedNumericDocValuesField("number", 20)));
102+
iw.addDocument(asList(new LongPoint("row", 1), new SortedNumericDocValuesField("number", 10)));
103+
};
104+
105+
testCase(LongPoint.newRangeQuery("row", 0, 2), docs, hdr -> {
106+
assertEquals(2L, hdr.state.getTotalCount());
107+
assertEquals(10.0d, hdr.percentile(randomDoubleBetween(1, 50, true)), 0.05d);
108+
});
109+
110+
testCase(LongPoint.newRangeQuery("row", 5, 10), docs, hdr -> {
111+
assertEquals(0L, hdr.state.getTotalCount());
112+
});
113+
}
114+
115+
private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
116+
Consumer<InternalHDRPercentiles> verify) throws IOException {
117+
try (Directory directory = newDirectory()) {
118+
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
119+
buildIndex.accept(indexWriter);
120+
}
121+
122+
try (IndexReader indexReader = DirectoryReader.open(directory)) {
123+
IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
124+
125+
PercentilesAggregationBuilder builder =
126+
new PercentilesAggregationBuilder("test").field("number").method(PercentilesMethod.HDR);
127+
128+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
129+
fieldType.setName("number");
130+
try (HDRPercentilesAggregator aggregator = createAggregator(builder, indexSearcher, fieldType)) {
131+
aggregator.preCollection();
132+
indexSearcher.search(query, aggregator);
133+
aggregator.postCollection();
134+
verify.accept((InternalHDRPercentiles) aggregator.buildAggregation(0L));
135+
}
136+
}
137+
}
138+
}
139+
}

‎core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorTests.java‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
21+
22+
import org.apache.lucene.document.LongPoint;
23+
import org.apache.lucene.document.NumericDocValuesField;
24+
import org.apache.lucene.document.SortedNumericDocValuesField;
25+
import org.apache.lucene.index.DirectoryReader;
26+
import org.apache.lucene.index.IndexReader;
27+
import org.apache.lucene.index.RandomIndexWriter;
28+
import org.apache.lucene.search.FieldValueQuery;
29+
import org.apache.lucene.search.IndexSearcher;
30+
import org.apache.lucene.search.MatchAllDocsQuery;
31+
import org.apache.lucene.search.Query;
32+
import org.apache.lucene.store.Directory;
33+
import org.elasticsearch.common.CheckedConsumer;
34+
import org.elasticsearch.index.mapper.MappedFieldType;
35+
import org.elasticsearch.index.mapper.NumberFieldMapper;
36+
import org.elasticsearch.search.aggregations.AggregatorTestCase;
37+
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder;
38+
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod;
39+
40+
import java.io.IOException;
41+
import java.util.function.Consumer;
42+
43+
import static java.util.Arrays.asList;
44+
import static java.util.Collections.singleton;
45+
46+
public class TDigestPercentilesAggregatorTests extends AggregatorTestCase {
47+
48+
public void testNoDocs() throws IOException {
49+
testCase(new MatchAllDocsQuery(), iw -> {
50+
// Intentionally not writing any docs
51+
}, tdigest -> {
52+
assertEquals(0L, tdigest.state.size());
53+
});
54+
}
55+
56+
public void testNoMatchingField() throws IOException {
57+
testCase(new MatchAllDocsQuery(), iw -> {
58+
iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
59+
iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 1)));
60+
}, tdigest -> {
61+
assertEquals(0L, tdigest.state.size());
62+
});
63+
}
64+
65+
public void testSomeMatchesSortedNumericDocValues() throws IOException {
66+
testCase(new FieldValueQuery("number"), iw -> {
67+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 8)));
68+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 5)));
69+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 3)));
70+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 2)));
71+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 1)));
72+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 1)));
73+
iw.addDocument(singleton(new SortedNumericDocValuesField("number", 0)));
74+
}, tdigest -> {
75+
assertEquals(7L, tdigest.state.size());
76+
assertEquals(7L, tdigest.state.centroidCount());
77+
assertEquals(4.0d, tdigest.percentile(75), 0.0d);
78+
assertEquals("4.0", tdigest.percentileAsString(75));
79+
assertEquals(2.0d, tdigest.percentile(50), 0.0d);
80+
assertEquals("2.0", tdigest.percentileAsString(50));
81+
assertEquals(1.0d, tdigest.percentile(20), 0.0d);
82+
assertEquals("1.0", tdigest.percentileAsString(20));
83+
});
84+
}
85+
86+
public void testSomeMatchesNumericDocValues() throws IOException {
87+
testCase(new FieldValueQuery("number"), iw -> {
88+
iw.addDocument(singleton(new NumericDocValuesField("number", 8)));
89+
iw.addDocument(singleton(new NumericDocValuesField("number", 5)));
90+
iw.addDocument(singleton(new NumericDocValuesField("number", 3)));
91+
iw.addDocument(singleton(new NumericDocValuesField("number", 2)));
92+
iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
93+
iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
94+
iw.addDocument(singleton(new NumericDocValuesField("number", 0)));
95+
}, tdigest -> {
96+
assertEquals(tdigest.state.size(), 7L);
97+
assertEquals(tdigest.state.centroidCount(), 7L);
98+
assertEquals(8.0d, tdigest.percentile(100), 0.0d);
99+
assertEquals("8.0", tdigest.percentileAsString(100));
100+
assertEquals(5.48d, tdigest.percentile(86), 0.0d);
101+
assertEquals("5.48", tdigest.percentileAsString(86));
102+
assertEquals(1.0d, tdigest.percentile(33), 0.0d);
103+
assertEquals("1.0", tdigest.percentileAsString(33));
104+
assertEquals(1.0d, tdigest.percentile(25), 0.0d);
105+
assertEquals("1.0", tdigest.percentileAsString(25));
106+
assertEquals(0.06d, tdigest.percentile(1), 0.0d);
107+
assertEquals("0.06", tdigest.percentileAsString(1));
108+
});
109+
}
110+
111+
public void testQueryFiltering() throws IOException {
112+
final CheckedConsumer<RandomIndexWriter, IOException> docs = iw -> {
113+
iw.addDocument(asList(new LongPoint("row", 7), new SortedNumericDocValuesField("number", 8)));
114+
iw.addDocument(asList(new LongPoint("row", 6), new SortedNumericDocValuesField("number", 5)));
115+
iw.addDocument(asList(new LongPoint("row", 5), new SortedNumericDocValuesField("number", 3)));
116+
iw.addDocument(asList(new LongPoint("row", 4), new SortedNumericDocValuesField("number", 2)));
117+
iw.addDocument(asList(new LongPoint("row", 3), new SortedNumericDocValuesField("number", 1)));
118+
iw.addDocument(asList(new LongPoint("row", 2), new SortedNumericDocValuesField("number", 1)));
119+
iw.addDocument(asList(new LongPoint("row", 1), new SortedNumericDocValuesField("number", 0)));
120+
};
121+
122+
testCase(LongPoint.newRangeQuery("row", 1, 4), docs, tdigest -> {
123+
assertEquals(4L, tdigest.state.size());
124+
assertEquals(4L, tdigest.state.centroidCount());
125+
assertEquals(2.0d, tdigest.percentile(100), 0.0d);
126+
assertEquals(1.0d, tdigest.percentile(50), 0.0d);
127+
assertEquals(0.75d, tdigest.percentile(25), 0.0d);
128+
});
129+
130+
testCase(LongPoint.newRangeQuery("row", 100, 110), docs, tdigest -> {
131+
assertEquals(0L, tdigest.state.size());
132+
assertEquals(0L, tdigest.state.centroidCount());
133+
});
134+
}
135+
136+
private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
137+
Consumer<InternalTDigestPercentiles> verify) throws IOException {
138+
try (Directory directory = newDirectory()) {
139+
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
140+
buildIndex.accept(indexWriter);
141+
}
142+
143+
try (IndexReader indexReader = DirectoryReader.open(directory)) {
144+
IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
145+
146+
PercentilesAggregationBuilder builder =
147+
new PercentilesAggregationBuilder("test").field("number").method(PercentilesMethod.TDIGEST);
148+
149+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
150+
fieldType.setName("number");
151+
try (TDigestPercentilesAggregator aggregator = createAggregator(builder, indexSearcher, fieldType)) {
152+
aggregator.preCollection();
153+
indexSearcher.search(query, aggregator);
154+
aggregator.postCollection();
155+
verify.accept((InternalTDigestPercentiles) aggregator.buildAggregation(0L));
156+
}
157+
}
158+
}
159+
}
160+
}

0 commit comments

Comments
 (0)