Skip to content

Commit 2a56a3a

Browse files
csoulioscsoulios
authored
Add tests to MedianAbsoluteDeviationAggregator (#54884) (#55282)
1 parent 6a0eebf commit 2a56a3a

File tree

3 files changed

+170
-61
lines changed

3 files changed

+170
-61
lines changed

‎server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java‎

Lines changed: 142 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -32,22 +32,37 @@
3232
import org.apache.lucene.search.Query;
3333
import org.apache.lucene.store.Directory;
3434
import org.elasticsearch.common.CheckedConsumer;
35+
import org.elasticsearch.common.settings.Settings;
3536
import org.elasticsearch.index.mapper.MappedFieldType;
3637
import org.elasticsearch.index.mapper.NumberFieldMapper;
38+
import org.elasticsearch.script.MockScriptEngine;
39+
import org.elasticsearch.script.Script;
40+
import org.elasticsearch.script.ScriptEngine;
41+
import org.elasticsearch.script.ScriptModule;
42+
import org.elasticsearch.script.ScriptService;
43+
import org.elasticsearch.script.ScriptType;
44+
import org.elasticsearch.search.aggregations.AggregationBuilder;
45+
import org.elasticsearch.search.aggregations.Aggregator;
3746
import org.elasticsearch.search.aggregations.AggregatorTestCase;
3847
import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
48+
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
49+
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
3950
import org.hamcrest.Description;
4051
import org.hamcrest.TypeSafeMatcher;
4152

4253
import java.io.IOException;
4354
import java.util.ArrayList;
4455
import java.util.Arrays;
56+
import java.util.Collections;
57+
import java.util.HashMap;
4558
import java.util.List;
59+
import java.util.Map;
4660
import java.util.function.Consumer;
4761
import java.util.function.Function;
4862
import java.util.stream.IntStream;
4963

5064
import static java.util.Collections.singleton;
65+
import static java.util.Collections.singletonList;
5166
import static org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregatorTests.ExactMedianAbsoluteDeviation.calculateMAD;
5267
import static org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregatorTests.IsCloseToRelative.closeToRelative;
5368
import static org.hamcrest.Matchers.equalTo;
@@ -56,6 +71,11 @@ public class MedianAbsoluteDeviationAggregatorTests extends AggregatorTestCase {
5671

5772
private static final int SAMPLE_MIN = -1000000;
5873
private static final int SAMPLE_MAX = 1000000;
74+
public static final String FIELD_NAME = "number";
75+
76+
/** Script to return the {@code _value} provided by aggs framework. */
77+
private static final String VALUE_SCRIPT = "_value";
78+
private static final String SINGLE_SCRIPT = "single";
5979

6080
private static <T extends IndexableField> CheckedConsumer<RandomIndexWriter, IOException> randomSample(
6181
int size,
@@ -96,10 +116,10 @@ public void testSomeMatchesSortedNumericDocValues() throws IOException {
96116
final int size = randomIntBetween(100, 1000);
97117
final List<Long> sample = new ArrayList<>(size);
98118
testCase(
99-
new DocValuesFieldExistsQuery("number"),
119+
new DocValuesFieldExistsQuery(FIELD_NAME),
100120
randomSample(size, point -> {
101121
sample.add(point);
102-
return singleton(new SortedNumericDocValuesField("number", point));
122+
return singleton(new SortedNumericDocValuesField(FIELD_NAME, point));
103123
}),
104124
agg -> {
105125
assertThat(agg.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(sample)));
@@ -112,10 +132,10 @@ public void testSomeMatchesNumericDocValues() throws IOException {
112132
final int size = randomIntBetween(100, 1000);
113133
final List<Long> sample = new ArrayList<>(size);
114134
testCase(
115-
new DocValuesFieldExistsQuery("number"),
135+
new DocValuesFieldExistsQuery(FIELD_NAME),
116136
randomSample(size, point -> {
117137
sample.add(point);
118-
return singleton(new NumericDocValuesField("number", point));
138+
return singleton(new NumericDocValuesField(FIELD_NAME, point));
119139
}),
120140
agg -> {
121141
assertThat(agg.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(sample)));
@@ -130,10 +150,10 @@ public void testQueryFiltering() throws IOException {
130150
final int[] sample = IntStream.rangeClosed(1, 1000).toArray();
131151
final int[] filteredSample = Arrays.stream(sample).filter(point -> point >= lowerRange && point <= upperRange).toArray();
132152
testCase(
133-
IntPoint.newRangeQuery("number", lowerRange, upperRange),
153+
IntPoint.newRangeQuery(FIELD_NAME, lowerRange, upperRange),
134154
writer -> {
135155
for (int point : sample) {
136-
writer.addDocument(Arrays.asList(new IntPoint("number", point), new SortedNumericDocValuesField("number", point)));
156+
writer.addDocument(Arrays.asList(new IntPoint(FIELD_NAME, point), new SortedNumericDocValuesField(FIELD_NAME, point)));
137157
}
138158
},
139159
agg -> {
@@ -145,10 +165,10 @@ public void testQueryFiltering() throws IOException {
145165

146166
public void testQueryFiltersAll() throws IOException {
147167
testCase(
148-
IntPoint.newRangeQuery("number", -1, 0),
168+
IntPoint.newRangeQuery(FIELD_NAME, -1, 0),
149169
writer -> {
150-
writer.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1)));
151-
writer.addDocument(Arrays.asList(new IntPoint("number", 2), new SortedNumericDocValuesField("number", 2)));
170+
writer.addDocument(Arrays.asList(new IntPoint(FIELD_NAME, 1), new SortedNumericDocValuesField(FIELD_NAME, 1)));
171+
writer.addDocument(Arrays.asList(new IntPoint(FIELD_NAME, 2), new SortedNumericDocValuesField(FIELD_NAME, 2)));
152172
},
153173
agg -> {
154174
assertThat(agg.getMedianAbsoluteDeviation(), equalTo(Double.NaN));
@@ -157,34 +177,110 @@ public void testQueryFiltersAll() throws IOException {
157177
);
158178
}
159179

180+
public void testUnmapped() throws IOException {
181+
MedianAbsoluteDeviationAggregationBuilder aggregationBuilder = new MedianAbsoluteDeviationAggregationBuilder("foo")
182+
.field(FIELD_NAME);
183+
184+
testCase(aggregationBuilder, new DocValuesFieldExistsQuery(FIELD_NAME), iw -> {
185+
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 7)));
186+
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 1)));
187+
}, agg -> {
188+
assertEquals(Double.NaN, agg.getMedianAbsoluteDeviation(),0);
189+
assertFalse(AggregationInspectionHelper.hasValue(agg));
190+
}, null);
191+
}
192+
193+
public void testUnmappedMissing() throws IOException {
194+
MedianAbsoluteDeviationAggregationBuilder aggregationBuilder = new MedianAbsoluteDeviationAggregationBuilder("foo")
195+
.field(FIELD_NAME)
196+
.missing(1234);
197+
198+
testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
199+
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
200+
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
201+
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
202+
}, agg -> {
203+
assertEquals(0, agg.getMedianAbsoluteDeviation(), 0);
204+
assertTrue(AggregationInspectionHelper.hasValue(agg));
205+
}, null);
206+
}
207+
208+
public void testValueScript() throws IOException {
209+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
210+
fieldType.setName(FIELD_NAME);
211+
fieldType.setHasDocValues(true);
212+
213+
MedianAbsoluteDeviationAggregationBuilder aggregationBuilder = new MedianAbsoluteDeviationAggregationBuilder("foo")
214+
.field(FIELD_NAME)
215+
.script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()));
216+
217+
final int size = randomIntBetween(100, 1000);
218+
final List<Long> sample = new ArrayList<>(size);
219+
testCase(aggregationBuilder,
220+
new MatchAllDocsQuery(),
221+
randomSample(size, point -> {
222+
sample.add(point);
223+
return singleton(new SortedNumericDocValuesField(FIELD_NAME, point));
224+
}),
225+
agg -> {
226+
assertThat(agg.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(sample)));
227+
assertTrue(AggregationInspectionHelper.hasValue(agg));
228+
}, fieldType);
229+
}
230+
231+
public void testSingleScript() throws IOException {
232+
MedianAbsoluteDeviationAggregationBuilder aggregationBuilder = new MedianAbsoluteDeviationAggregationBuilder("foo")
233+
.script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, SINGLE_SCRIPT, Collections.emptyMap()));
234+
235+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
236+
fieldType.setName(FIELD_NAME);
237+
238+
final int size = randomIntBetween(100, 1000);
239+
final List<Long> sample = new ArrayList<>(size);
240+
testCase(aggregationBuilder,
241+
new MatchAllDocsQuery(),
242+
iw -> {
243+
for (int i = 0; i < 10; i++) {
244+
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, i + 1)));
245+
}
246+
},
247+
agg -> {
248+
assertEquals(0, agg.getMedianAbsoluteDeviation(), 0);
249+
assertTrue(AggregationInspectionHelper.hasValue(agg));
250+
}, fieldType);
251+
}
252+
160253
private void testCase(Query query,
161-
CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
254+
CheckedConsumer<RandomIndexWriter,
255+
IOException> buildIndex,
162256
Consumer<InternalMedianAbsoluteDeviation> verify) throws IOException {
257+
MedianAbsoluteDeviationAggregationBuilder builder = new MedianAbsoluteDeviationAggregationBuilder("mad")
258+
.field(FIELD_NAME)
259+
.compression(randomDoubleBetween(20, 1000, true));
260+
261+
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
262+
fieldType.setName(FIELD_NAME);
263+
264+
testCase(builder, query, buildIndex, verify, fieldType);
265+
}
163266

267+
private void testCase(MedianAbsoluteDeviationAggregationBuilder aggregationBuilder, Query query,
268+
CheckedConsumer<RandomIndexWriter, IOException> indexer,
269+
Consumer<InternalMedianAbsoluteDeviation> verify, MappedFieldType fieldType) throws IOException {
164270
try (Directory directory = newDirectory()) {
165271
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
166-
buildIndex.accept(indexWriter);
272+
indexer.accept(indexWriter);
167273
}
168274

169275
try (IndexReader indexReader = DirectoryReader.open(directory)) {
170276
IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
171-
172-
MedianAbsoluteDeviationAggregationBuilder builder = new MedianAbsoluteDeviationAggregationBuilder("mad")
173-
.field("number")
174-
.compression(randomDoubleBetween(20, 1000, true));
175-
176-
MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
177-
fieldType.setName("number");
178-
179-
MedianAbsoluteDeviationAggregator aggregator = createAggregator(builder, indexSearcher, fieldType);
277+
Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
180278
aggregator.preCollection();
181279
indexSearcher.search(query, aggregator);
182280
aggregator.postCollection();
183-
184281
verify.accept((InternalMedianAbsoluteDeviation) aggregator.buildAggregation(0L));
185282
}
186283
}
187-
188284
}
189285

190286
public static class IsCloseToRelative extends TypeSafeMatcher<Double> {
@@ -271,6 +367,30 @@ private static double calculateMedian(double[] sample) {
271367
}
272368
return median;
273369
}
370+
}
371+
372+
@Override
373+
protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
374+
return singletonList(CoreValuesSourceType.NUMERIC);
375+
}
376+
377+
@Override
378+
protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
379+
return new MedianAbsoluteDeviationAggregationBuilder("foo").field(fieldName);
380+
}
381+
382+
@Override
383+
protected ScriptService getMockScriptService() {
384+
Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
385+
386+
scripts.put(VALUE_SCRIPT, vars -> ((Number) vars.get("_value")).doubleValue() + 1);
387+
scripts.put(SINGLE_SCRIPT, vars -> 1);
388+
389+
MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME,
390+
scripts,
391+
Collections.emptyMap());
392+
Map<String, ScriptEngine> engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine);
274393

394+
return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
275395
}
276396
}

‎server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -182,20 +182,7 @@ public void testEmptyAggregation() throws Exception {
182182

183183
@Override
184184
public void testUnmapped() throws Exception {
185-
final SearchResponse response = client()
186-
.prepareSearch("idx_unmapped")
187-
.setQuery(matchAllQuery())
188-
.addAggregation(
189-
randomBuilder()
190-
.field("value"))
191-
.get();
192-
193-
assertHitCount(response, 0);
194-
195-
final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
196-
assertThat(mad, notNullValue());
197-
assertThat(mad.getName(), is("mad"));
198-
assertThat(mad.getMedianAbsoluteDeviation(), is(Double.NaN));
185+
// Test moved to MedianAbsoluteDeviationAggregatorTests.testUnmapped()
199186
}
200187

201188
@Override

‎server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java‎

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,9 @@ public void testUnmappedMissingString() throws IOException {
219219
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
220220
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
221221
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
222-
}, card -> {
223-
assertEquals(3, card.getValue(), 0);
224-
assertTrue(AggregationInspectionHelper.hasValue(card));
222+
}, valueCount -> {
223+
assertEquals(3, valueCount.getValue(), 0);
224+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
225225
}, null);
226226
}
227227

@@ -233,9 +233,9 @@ public void testUnmappedMissingNumber() throws IOException {
233233
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
234234
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
235235
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
236-
}, card -> {
237-
assertEquals(3, card.getValue(), 0);
238-
assertTrue(AggregationInspectionHelper.hasValue(card));
236+
}, valueCount -> {
237+
assertEquals(3, valueCount.getValue(), 0);
238+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
239239
}, null);
240240
}
241241

@@ -247,9 +247,9 @@ public void testUnmappedMissingGeoPoint() throws IOException {
247247
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
248248
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
249249
iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
250-
}, card -> {
251-
assertEquals(3, card.getValue(), 0);
252-
assertTrue(AggregationInspectionHelper.hasValue(card));
250+
}, valueCount -> {
251+
assertEquals(3, valueCount.getValue(), 0);
252+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
253253
}, null);
254254
}
255255

@@ -288,15 +288,15 @@ public void testValueScriptNumber() throws IOException {
288288
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 7)));
289289
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 8)));
290290
iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 9)));
291-
}, card -> {
292-
assertEquals(3, card.getValue(), 0);
293-
assertTrue(AggregationInspectionHelper.hasValue(card));
291+
}, valueCount -> {
292+
assertEquals(3, valueCount.getValue(), 0);
293+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
294294
}, fieldType);
295295
}
296296

297297
public void testSingleScriptNumber() throws IOException {
298298
ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null)
299-
.field(FIELD_NAME);
299+
.script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, SINGLE_SCRIPT, Collections.emptyMap()));
300300

301301
MappedFieldType fieldType = createMappedFieldType(ValueType.NUMERIC);
302302
fieldType.setName(FIELD_NAME);
@@ -317,10 +317,11 @@ public void testSingleScriptNumber() throws IOException {
317317
doc.add(new SortedNumericDocValuesField(FIELD_NAME, 1));
318318
doc.add(new SortedNumericDocValuesField(FIELD_NAME, 1));
319319
iw.addDocument(doc);
320-
}, card -> {
321-
// note: this is 6, even though the script returns a single value. ValueCount does not de-dedupe
322-
assertEquals(6, card.getValue(), 0);
323-
assertTrue(AggregationInspectionHelper.hasValue(card));
320+
}, valueCount -> {
321+
// Note: The field values won't be taken into account. The script will only be called
322+
// once per document, and only expect a count of 3
323+
assertEquals(3, valueCount.getValue(), 0);
324+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
324325
}, fieldType);
325326
}
326327

@@ -337,15 +338,15 @@ public void testValueScriptString() throws IOException {
337338
iw.addDocument(singleton(new SortedDocValuesField(FIELD_NAME, new BytesRef("1"))));
338339
iw.addDocument(singleton(new SortedDocValuesField(FIELD_NAME, new BytesRef("2"))));
339340
iw.addDocument(singleton(new SortedDocValuesField(FIELD_NAME, new BytesRef("3"))));
340-
}, card -> {
341-
assertEquals(3, card.getValue(), 0);
342-
assertTrue(AggregationInspectionHelper.hasValue(card));
341+
}, valueCount -> {
342+
assertEquals(3, valueCount.getValue(), 0);
343+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
343344
}, fieldType);
344345
}
345346

346347
public void testSingleScriptString() throws IOException {
347348
ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null)
348-
.field(FIELD_NAME);
349+
.script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, SINGLE_SCRIPT, Collections.emptyMap()));
349350

350351
MappedFieldType fieldType = createMappedFieldType(ValueType.STRING);
351352
fieldType.setName(FIELD_NAME);
@@ -367,10 +368,11 @@ public void testSingleScriptString() throws IOException {
367368
doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("5")));
368369
doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef("6")));
369370
iw.addDocument(doc);
370-
}, card -> {
371-
// note: this is 6, even though the script returns a single value. ValueCount does not de-dedupe
372-
assertEquals(6, card.getValue(), 0);
373-
assertTrue(AggregationInspectionHelper.hasValue(card));
371+
}, valueCount -> {
372+
// Note: The field values won't be taken into account. The script will only be called
373+
// once per document, and only expect a count of 3
374+
assertEquals(3, valueCount.getValue(), 0);
375+
assertTrue(AggregationInspectionHelper.hasValue(valueCount));
374376
}, fieldType);
375377
}
376378

0 commit comments

Comments
 (0)