From 9e22e099c4ffe39c2379876eae6f1454875a98b0 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 20 Feb 2020 14:08:25 -0500 Subject: [PATCH 1/2] Comprehensively test supported/unsupported field type:agg combinations (#52493) This adds a test to AggregatorTestCase that allows us to programmatically verify that an aggregator supports or does not support a particular field type. It fetches the list of registered field type parsers, creates a MappedFieldType from the parser and then attempts to run a basic agg against the field. A supplied list of supported VSTypes are then compared against the output (success or exception) and suceeds or fails the test accordingly. Co-Authored-By: Mark Tozzi * Skip fields that are not aggregatable --- .../index/mapper/ScaledFloatFieldMapper.java | 7 + .../join/mapper/MetaJoinFieldMapper.java | 7 + .../join/mapper/ParentIdFieldMapper.java | 7 + .../join/mapper/ParentJoinFieldMapper.java | 7 + .../ICUCollationKeywordFieldMapper.java | 7 + .../mapper/murmur3/Murmur3FieldMapper.java | 7 + .../index/mapper/BinaryFieldMapper.java | 7 + .../index/mapper/BooleanFieldMapper.java | 7 + .../index/mapper/DateFieldMapper.java | 10 +- .../index/mapper/GeoPointFieldMapper.java | 10 +- .../index/mapper/IdFieldMapper.java | 8 + .../index/mapper/IndexFieldMapper.java | 8 + .../index/mapper/IpFieldMapper.java | 7 + .../index/mapper/KeywordFieldMapper.java | 7 + .../index/mapper/MappedFieldType.java | 11 + .../index/mapper/NumberFieldMapper.java | 7 + .../index/mapper/RangeFieldMapper.java | 15 +- .../index/mapper/SeqNoFieldMapper.java | 6 + .../index/mapper/TextFieldMapper.java | 7 + .../index/mapper/TypeFieldMapper.java | 7 + .../aggregations/bucket/IpRangeTests.java | 30 +-- .../bucket/range/IpRangeAggregatorTests.java | 18 -- .../bucket/terms/TermsAggregatorTests.java | 14 ++ .../aggregations/AggregatorTestCase.java | 229 +++++++++++++++++- .../org/elasticsearch/test/ESTestCase.java | 19 +- .../mapper/HistogramFieldMapper.java | 8 + .../mapper/FlatObjectFieldMapper.java | 7 + .../mapper/DenseVectorFieldMapper.java | 7 + 28 files changed, 434 insertions(+), 57 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index aec9208df5df4..81c092f3a2de6 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -62,6 +62,8 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.math.BigDecimal; @@ -295,6 +297,11 @@ public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fiel }; } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 388d4ca833ff4..f1b75672ccc4b 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -31,6 +31,8 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.List; @@ -95,6 +97,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 8130acac1af72..7dc43962ad9c4 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -41,6 +41,8 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Collection; @@ -118,6 +120,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 331746b4276de..4c7c14893af12 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -43,6 +43,8 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -227,6 +229,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 883468941a5d8..57a0dba31f41f 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -46,6 +46,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -139,6 +141,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected BytesRef indexedValueForSearch(Object value) { if (value == null) { diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index cf2b05fe54d90..b6c49a246f433 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -39,6 +39,8 @@ import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.List; @@ -124,6 +126,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override public Query existsQuery(QueryShardContext context) { return new DocValuesFieldExistsQuery(name()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 7a5bd97770297..ea8d407f731f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -40,6 +40,8 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -136,6 +138,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new BytesBinaryDVIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Query existsQuery(QueryShardContext context) { if (hasDocValues()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index caf8baac24da1..009848df1d8a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -40,6 +40,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -189,6 +191,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(NumericType.BOOLEAN); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { if (format != null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 4bd119ff39e64..3fe1df76495d0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -54,6 +54,8 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.DateTimeException; @@ -73,6 +75,7 @@ public final class DateFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "date"; + public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos"; public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); public static class Defaults { @@ -101,7 +104,7 @@ public long parsePointAsMillis(byte[] value) { return LongPoint.decodeDimension(value, 0); } }, - NANOSECONDS("date_nanos", NumericType.DATE_NANOSECONDS) { + NANOSECONDS(DATE_NANOS_CONTENT_TYPE, NumericType.DATE_NANOSECONDS) { @Override public long convert(Instant instant) { return toLong(instant); @@ -540,6 +543,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(resolution.numericType()); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override public Object valueForDisplay(Object value) { Long val = (Long) value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 6ff18ecc9485f..d0af10336a61f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -41,6 +41,8 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.VectorGeoPointShapeQueryProcessor; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -243,6 +245,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new AbstractLatLonPointDVIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.GEOPOINT; + } + @Override public Query existsQuery(QueryShardContext context) { if (hasDocValues()) { @@ -254,8 +261,7 @@ public Query existsQuery(QueryShardContext context) { @Override public Query termQuery(Object value, QueryShardContext context) { - throw new QueryShardException(context, - "Geo fields do not support exact searching, use dedicated geo queries instead: [" + throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: [" + name() + "]"); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index b582b4f75ea83..58c3b17714980 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -50,6 +50,8 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Arrays; @@ -157,6 +159,12 @@ public Query termsQuery(List values, QueryShardContext context) { return new TermInSetQuery(name(), bytesRefs); } + @Override + public ValuesSourceType getValuesSourceType() { + // TODO: should this even exist? Is aggregating on the ID field valid? + return CoreValuesSourceType.BYTES; + } + @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { if (indexOptions() == IndexOptions.NONE) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 603b8b22216aa..018a5df4ea271 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -27,6 +27,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.List; @@ -110,6 +112,12 @@ protected boolean matches(String pattern, QueryShardContext context) { public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new ConstantIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName); } + + @Override + public ValuesSourceType getValuesSourceType() { + // TODO: Should Index fields be aggregatable? What even is an IndexField? + return CoreValuesSourceType.BYTES; + } } private IndexFieldMapper(Settings indexSettings, MappedFieldType existing) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 2b52e42ffe558..c1cf50a56cde5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -44,6 +44,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.net.InetAddress; @@ -294,6 +296,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().scriptFunction(IpScriptDocValues::new); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 778fa3450a516..b83427a53a3ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -43,6 +43,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Iterator; @@ -272,6 +274,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index fcaaac99511e3..94775740bad30 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -50,6 +50,7 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -118,6 +119,16 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]"); } + /** + * Returns the {@link ValuesSourceType} which supports this field type. This is tightly coupled to field data and aggregations support, + * so any implementation that returns a value from {@link MappedFieldType#fielddataBuilder} should also return a value from here. + * + * @return The appropriate {@link ValuesSourceType} for this field type. + */ + public ValuesSourceType getValuesSourceType() { + throw new IllegalArgumentException("Aggregations are not supported on field [" + name() + "] of type [" + typeName() + "]"); + } + @Override public boolean equals(Object o) { if (!super.equals(o)) return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 0a473bd189e3b..fa7055cadd2a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -55,6 +55,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -958,6 +960,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(type.numericType()); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 2c3af97ac095a..6dee473e9a06a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -46,6 +46,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.net.InetAddress; @@ -93,14 +95,6 @@ public RangeFieldType fieldType() { return (RangeFieldType)fieldType; } - @Override - public Builder docValues(boolean docValues) { - if (docValues) { - throw new IllegalArgumentException("field [" + name + "] does not currently support " + TypeParsers.DOC_VALUES); - } - return super.docValues(docValues); - } - public Builder coerce(boolean coerce) { this.coerce = coerce; return builder; @@ -249,6 +243,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().setRangeType(rangeType); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.RANGE; + } + @Override public String typeName() { return rangeType.name; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 8c032402b5090..0abcfcda8aca4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -38,6 +38,8 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.List; @@ -219,6 +221,10 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } } public SeqNoFieldMapper(Settings indexSettings) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index f9b1704e34fcb..8b9df6db76a92 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -69,6 +69,8 @@ import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.query.IntervalBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -764,6 +766,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new PagedBytesIndexFieldData.Builder(fielddataMinFrequency, fielddataMaxFrequency, fielddataMinSegmentSize); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public void checkCompatibility(MappedFieldType other, List conflicts) { super.checkCompatibility(other, conflicts); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 1795e4a629b8b..34d31d6f717cf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -46,6 +46,8 @@ import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.support.QueryParsers; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Arrays; @@ -119,6 +121,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new ConstantIndexFieldData.Builder(typeFunction); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public boolean isSearchable() { return true; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java index 56587c78708c1..0bf34f6e4a3e3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java @@ -23,27 +23,8 @@ import org.elasticsearch.search.aggregations.BaseAggregationTestCase; import org.elasticsearch.search.aggregations.bucket.range.IpRangeAggregationBuilder; -import java.net.InetAddress; -import java.net.UnknownHostException; - public class IpRangeTests extends BaseAggregationTestCase { - private static String randomIp(boolean v4) { - try { - if (v4) { - byte[] ipv4 = new byte[4]; - random().nextBytes(ipv4); - return NetworkAddress.format(InetAddress.getByAddress(ipv4)); - } else { - byte[] ipv6 = new byte[16]; - random().nextBytes(ipv6); - return NetworkAddress.format(InetAddress.getByAddress(ipv6)); - } - } catch (UnknownHostException e) { - throw new AssertionError(); - } - } - @Override protected IpRangeAggregationBuilder createTestAggregatorBuilder() { int numRanges = randomIntBetween(1, 10); @@ -62,16 +43,17 @@ protected IpRangeAggregationBuilder createTestAggregatorBuilder() { } else { prefixLength = randomInt(128); } - factory.addMaskRange(key, randomIp(v4) + "/" + prefixLength); + factory.addMaskRange(key, NetworkAddress.format(randomIp(v4)) + "/" + prefixLength); break; case 1: - factory.addUnboundedFrom(key, randomIp(randomBoolean())); + factory.addUnboundedFrom(key, NetworkAddress.format(randomIp(randomBoolean()))); break; case 2: - factory.addUnboundedTo(key, randomIp(randomBoolean())); + factory.addUnboundedTo(key, NetworkAddress.format(randomIp(randomBoolean()))); break; case 3: - factory.addRange(key, randomIp(randomBoolean()), randomIp(randomBoolean())); + v4 = randomBoolean(); + factory.addRange(key, NetworkAddress.format(randomIp(v4)), NetworkAddress.format(randomIp(v4))); break; default: fail(); @@ -82,7 +64,7 @@ protected IpRangeAggregationBuilder createTestAggregatorBuilder() { factory.keyed(randomBoolean()); } if (randomBoolean()) { - factory.missing(randomIp(randomBoolean())); + factory.missing(NetworkAddress.format(randomIp(randomBoolean()))); } return factory; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java index f759d30cbbde5..f96e7b01f25ca 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java @@ -35,29 +35,11 @@ import org.elasticsearch.search.aggregations.AggregatorTestCase; import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.Arrays; import java.util.Comparator; public class IpRangeAggregatorTests extends AggregatorTestCase { - private static InetAddress randomIp(boolean v4) { - try { - if (v4) { - byte[] ipv4 = new byte[4]; - random().nextBytes(ipv4); - return InetAddress.getByAddress(ipv4); - } else { - byte[] ipv6 = new byte[16]; - random().nextBytes(ipv6); - return InetAddress.getByAddress(ipv6); - } - } catch (UnknownHostException e) { - throw new AssertionError(); - } - } - - private static boolean isInRange(BytesRef value, BytesRef from, BytesRef to) { if ((to == null || to.compareTo(value) > 0) && (from == null || from.compareTo(value) <= 0)) { return true; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index f11c41a13889a..57b255a157382 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -81,7 +81,9 @@ import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.test.geo.RandomGeoGenerator; @@ -98,6 +100,7 @@ import java.util.function.BiFunction; import java.util.function.Function; +import static java.util.Arrays.asList; import static org.elasticsearch.index.mapper.SeqNoFieldMapper.PRIMARY_TERM_NAME; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.bucketScript; @@ -123,6 +126,17 @@ protected A createAggregator(AggregationBuilder aggregati } } + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new TermsAggregationBuilder("foo", ValueType.STRING).field(fieldName); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return Collections.unmodifiableList(asList(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.BYTES)); + } + public void testGlobalOrdinalsExecutionHint() throws Exception { randomizeAggregatorImpl = false; diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index c5177da5d9ca3..89cd2c0e56add 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -18,20 +18,29 @@ */ package org.elasticsearch.search.aggregations; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.AssertingDirectoryReader; import org.apache.lucene.index.CompositeReaderContext; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCache; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -51,24 +60,40 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.BinaryFieldMapper; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldAliasMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.ObjectMapper.Nested; +import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; +import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.subphase.FetchDocValuesPhase; import org.elasticsearch.search.fetch.subphase.FetchSourcePhase; @@ -91,6 +116,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.singleton; import static org.elasticsearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; @@ -108,6 +134,21 @@ public abstract class AggregatorTestCase extends ESTestCase { private List releasables = new ArrayList<>(); private static final String TYPE_NAME = "type"; + // A list of field types that should not be tested, or are not currently supported + private static List TYPE_TEST_BLACKLIST; + + static { + List blacklist = new ArrayList<>(); + blacklist.add(ObjectMapper.CONTENT_TYPE); // Cannot aggregate objects + blacklist.add(GeoShapeFieldMapper.CONTENT_TYPE); // Cannot aggregate geoshapes (yet) + blacklist.add(TextFieldMapper.CONTENT_TYPE); // TODO Does not support doc values, but does support FD, needs a lot of mocking + blacklist.add(ObjectMapper.NESTED_CONTENT_TYPE); // TODO support for nested + blacklist.add(CompletionFieldMapper.CONTENT_TYPE); // TODO support completion + blacklist.add(FieldAliasMapper.CONTENT_TYPE); // TODO support alias + TYPE_TEST_BLACKLIST = blacklist; + } + + /** * Allows subclasses to provide alternate names for the provided field type, which * can be useful when testing aggregations on field aliases. @@ -116,9 +157,8 @@ protected Map getFieldAliases(MappedFieldType... fieldT return Collections.emptyMap(); } - private void registerFieldTypes(SearchContext searchContext, - MapperService mapperService, - Map fieldNameToType) { + private static void registerFieldTypes(SearchContext searchContext, MapperService mapperService, + Map fieldNameToType) { for (Map.Entry entry : fieldNameToType.entrySet()) { String fieldName = entry.getKey(); MappedFieldType fieldType = entry.getValue(); @@ -521,6 +561,189 @@ protected static IndexReader maybeWrapReaderEs(DirectoryReader reader) throws IO } } + /** + * Implementors should return a list of {@link ValuesSourceType} that the aggregator supports. + * This is used to test the matrix of supported/unsupported field types against the aggregator + * and verify it works (or doesn't) as expected. + * + * If this method is implemented, {@link AggregatorTestCase#createAggBuilderForTypeTest(MappedFieldType, String)} + * should be implemented as well. + * + * @return list of supported ValuesSourceTypes + */ + protected List getSupportedValuesSourceTypes() { + // If aggs don't override this method, an empty list allows the test to be skipped. + // Once all aggs implement this method we should make it abstract and not allow skipping. + return Collections.emptyList(); + } + + /** + * This method is invoked each time a field type is tested in {@link AggregatorTestCase#testSupportedFieldTypes()}. + * The field type and name are provided, and the implementor is expected to return an AggBuilder accordingly. + * The AggBuilder should be returned even if the aggregation does not support the field type, because + * the test will check if an exception is thrown in that case. + * + * The list of supported types are provided by {@link AggregatorTestCase#getSupportedValuesSourceTypes()}, + * which must also be implemented. + * + * @param fieldType the type of the field that will be tested + * @param fieldName the name of the field that will be test + * @return an aggregation builder to test against the field + */ + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + throw new UnsupportedOperationException("If getSupportedValuesSourceTypes() is implemented, " + + "createAggBuilderForTypeTest() must be implemented as well."); + } + + /** + * This test will validate that an aggregator succeeds or fails to run against all the field types + * that are registered in {@link IndicesModule} (e.g. all the core field types). An aggregator + * is provided by the implementor class, and it is executed against each field type in turn. If + * an exception is thrown when the field is supported, that will fail the test. Similarly, if + * an exception _is not_ thrown when a field is unsupported, that will also fail the test. + * + * Exception types/messages are not currently checked, just presence/absence of an exception. + */ + public void testSupportedFieldTypes() throws IOException { + MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry(); + Settings settings = Settings.builder().put("index.version.created", Version.CURRENT.id).build(); + String fieldName = "typeTestFieldName"; + List supportedVSTypes = getSupportedValuesSourceTypes(); + + if (supportedVSTypes.isEmpty()) { + // If the test says it doesn't support any VStypes, it has not been converted yet so skip + return; + } else if (supportedVSTypes.contains(CoreValuesSourceType.ANY)) { + throw new IllegalArgumentException("Tests should not specify CoreValuesSourceType.ANY as a supported ValuesSourceType, " + + "but should instead list the concrete ValuesSourceTypes that are supported"); + } + + for (Map.Entry mappedType : mapperRegistry.getMapperParsers().entrySet()) { + + // Some field types should not be tested, or require more work and are not ready yet + if (TYPE_TEST_BLACKLIST.contains(mappedType.getKey())) { + continue; + } + + Map source = new HashMap<>(); + source.put("type", mappedType.getKey()); + source.put("doc_values", "true"); + + Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext()); + FieldMapper mapper = (FieldMapper) builder.build(new BuilderContext(settings, new ContentPath())); + + MappedFieldType fieldType = mapper.fieldType(); + + // Non-aggregatable fields are not testable (they will throw an error on all aggs anyway), so skip + if (fieldType.isAggregatable() == false) { + continue; + } + + try (Directory directory = newDirectory()) { + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + writeTestDoc(fieldType, fieldName, indexWriter); + indexWriter.close(); + + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, fieldName); + + // TODO in the future we can make this more explicit with expectThrows(), when the exceptions are standardized + try { + searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); + if (supportedVSTypes.contains(fieldType.getValuesSourceType()) == false) { + fail("Aggregator [" + aggregationBuilder.getType() + "] should not support field type [" + + fieldType.typeName() + "] but executing against the field did not throw an excetion"); + } + } catch (Exception e) { + if (supportedVSTypes.contains(fieldType.getValuesSourceType())) { + fail("Aggregator [" + aggregationBuilder.getType() + "] supports field type [" + + fieldType.typeName() + "] but executing against the field threw an exception: [" + e.getMessage() + "]"); + } + } + } + } + } + } + + /** + * Helper method to write a single document with a single value specific to the requested fieldType. + * + * Throws an exception if it encounters an unknown field type, to prevent new ones from sneaking in without + * being tested. + */ + private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomIndexWriter iw) throws IOException { + + if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.NUMERIC)) { + // TODO note: once VS refactor adds DATE/BOOLEAN, this conditional will go away + if (fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE) + || fieldType.typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) { + iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomNonNegativeLong()))); + } else if (fieldType.typeName().equals(BooleanFieldMapper.CONTENT_TYPE)) { + iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomBoolean() ? 0 : 1))); + } else { + iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomLong()))); + } + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.BYTES)) { + if (fieldType.typeName().equals(BinaryFieldMapper.CONTENT_TYPE)) { + iw.addDocument(singleton(new BinaryFieldMapper.CustomBinaryDocValuesField(fieldName, new BytesRef("a").bytes))); + } else if (fieldType.typeName().equals(IpFieldMapper.CONTENT_TYPE)) { + // TODO note: once VS refactor adds IP, this conditional will go away + boolean v4 = randomBoolean(); + iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(randomIp(v4)))))); + } else { + iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef("a")))); + } + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.RANGE)) { + Object start; + Object end; + RangeType rangeType; + + if (fieldType.typeName().equals(RangeType.DOUBLE.typeName())) { + start = randomDouble(); + end = RangeType.DOUBLE.nextUp(start); + rangeType = RangeType.DOUBLE; + } else if (fieldType.typeName().equals(RangeType.FLOAT.typeName())) { + start = randomFloat(); + end = RangeType.FLOAT.nextUp(start); + rangeType = RangeType.DOUBLE; + } else if (fieldType.typeName().equals(RangeType.IP.typeName())) { + boolean v4 = randomBoolean(); + start = randomIp(v4); + end = RangeType.IP.nextUp(start); + rangeType = RangeType.IP; + } else if (fieldType.typeName().equals(RangeType.LONG.typeName())) { + start = randomLong(); + end = RangeType.LONG.nextUp(start); + rangeType = RangeType.LONG; + } else if (fieldType.typeName().equals(RangeType.INTEGER.typeName())) { + start = randomInt(); + end = RangeType.INTEGER.nextUp(start); + rangeType = RangeType.INTEGER; + } else if (fieldType.typeName().equals(RangeType.DATE.typeName())) { + start = randomNonNegativeLong(); + end = RangeType.DATE.nextUp(start); + rangeType = RangeType.DATE; + } else { + throw new IllegalStateException("Unknown type of range [" + fieldType.typeName() + "]"); + } + + final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, start, end, true, true); + iw.addDocument(singleton(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(range))))); + + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.GEOPOINT)) { + iw.addDocument(singleton(new LatLonDocValuesField(fieldName, randomDouble(), randomDouble()))); + } else { + throw new IllegalStateException("Unknown field type [" + fieldType.typeName() + "]"); + } + } + + private class MockParserContext extends Mapper.TypeParser.ParserContext { + MockParserContext() { + super(null, null, null, null, null); + } + } + @After private void cleanupReleasables() { Releasables.close(releasables); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 69adb6ec3f24f..4d7e7103c42fd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -29,7 +29,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; - import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -119,6 +118,8 @@ import java.io.IOException; import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.nio.file.Path; import java.time.ZoneId; import java.util.ArrayList; @@ -1434,4 +1435,20 @@ protected static int getBasePort() { assert startAt >= 0 : "Unexpected test worker Id, resulting port range would be negative"; return 10300 + (startAt * 100); } + + protected static InetAddress randomIp(boolean v4) { + try { + if (v4) { + byte[] ipv4 = new byte[4]; + random().nextBytes(ipv4); + return InetAddress.getByAddress(ipv4); + } else { + byte[] ipv6 = new byte[16]; + random().nextBytes(ipv6); + return InetAddress.getByAddress(ipv6); + } + } catch (UnknownHostException e) { + throw new AssertionError(); + } + } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 97d8b80225a26..6b77018c11638 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -53,6 +53,8 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Iterator; @@ -276,6 +278,12 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu }; } + @Override + public ValuesSourceType getValuesSourceType() { + // TODO: Histogram ValuesSourceType should move into this plugin. + return CoreValuesSourceType.HISTOGRAM; + } + @Override public Query existsQuery(QueryShardContext context) { if (hasDocValues()) { diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 92c8a071c88fe..9bbb2a5d6d6c1 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -55,6 +55,8 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Iterator; @@ -349,6 +351,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); return new KeyedFlatObjectFieldData.Builder(key); } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } } /** diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 7656ae0b9c114..a3e7b23b87199 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -27,6 +27,8 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.xpack.vectors.query.VectorDVIndexFieldData; import java.io.IOException; @@ -150,6 +152,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new VectorDVIndexFieldData.Builder(true); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new UnsupportedOperationException( From bc90d74fa8225ac89a25b68a12d00788e04d33c1 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Mon, 24 Feb 2020 14:55:52 -0500 Subject: [PATCH 2/2] Use newIndexSearcher() to avoid incompatible readers (#52723) Lucene's `newSearcher()` can generate readers like ParallelCompositeReader which we can't use. We need to instead use our helper `newIndexSearcher` --- .../elasticsearch/search/aggregations/AggregatorTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 89cd2c0e56add..0ceec0f23d3f3 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -645,7 +645,7 @@ public void testSupportedFieldTypes() throws IOException { indexWriter.close(); try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, fieldName); // TODO in the future we can make this more explicit with expectThrows(), when the exceptions are standardized