diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java index cc39bbe880544..0963dacd403a2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java @@ -30,12 +30,10 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; @@ -52,14 +50,14 @@ * {@linkplain AggregationBuilder#rewrite(QueryRewriteContext)}, or * {@linkplain AbstractAggregationBuilder#build(QueryShardContext, AggregatorFactory)}. */ -public class StringStatsAggregationBuilder extends ValuesSourceAggregationBuilder { +public class StringStatsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "string_stats"; private static final ParseField SHOW_DISTRIBUTION_FIELD = new ParseField("show_distribution"); private boolean showDistribution = false; public StringStatsAggregationBuilder(String name) { - super(name, CoreValuesSourceType.BYTES, ValueType.STRING); + super(name); } /** @@ -71,6 +69,11 @@ public StringStatsAggregationBuilder showDistribution(boolean showDistribution) return this; } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public String getType() { return NAME; @@ -87,7 +90,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException { } @Override - protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { throw new UnsupportedOperationException(); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 346a6310c5d75..1360b58e0b1ec 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1077,7 +1077,8 @@ public static SearchSourceBuilder createTestSearchSourceBuilder() { searchSourceBuilder.query(new TermQueryBuilder(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10))); } if (randomBoolean()) { - searchSourceBuilder.aggregation(new TermsAggregationBuilder(randomAlphaOfLengthBetween(3, 10), ValueType.STRING) + searchSourceBuilder.aggregation(new TermsAggregationBuilder(randomAlphaOfLengthBetween(3, 10)) + .userValueTypeHint(ValueType.STRING) .field(randomAlphaOfLengthBetween(3, 10))); } if (randomBoolean()) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index b4b8787f86418..eac96aeed272d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -257,7 +257,8 @@ public void testSearchMatchQuery() throws IOException { public void testSearchWithTermsAgg() throws IOException { SearchRequest searchRequest = new SearchRequest(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.aggregation(new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword")); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("agg1").userValueTypeHint(ValueType.STRING) + .field("type.keyword")); searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); @@ -349,7 +350,7 @@ public void testSearchWithRangeAgg() throws IOException { public void testSearchWithTermsAndRangeAgg() throws IOException { SearchRequest searchRequest = new SearchRequest("index"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - TermsAggregationBuilder agg = new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword"); + TermsAggregationBuilder agg = new TermsAggregationBuilder("agg1").userValueTypeHint(ValueType.STRING).field("type.keyword"); agg.subAggregation(new RangeAggregationBuilder("subagg").field("num") .addRange("first", 0, 30).addRange("second", 31, 200)); searchSourceBuilder.aggregation(agg); @@ -403,7 +404,7 @@ public void testSearchWithTermsAndRangeAgg() throws IOException { public void testSearchWithTermsAndWeightedAvg() throws IOException { SearchRequest searchRequest = new SearchRequest("index"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - TermsAggregationBuilder agg = new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword"); + TermsAggregationBuilder agg = new TermsAggregationBuilder("agg1").userValueTypeHint(ValueType.STRING).field("type.keyword"); agg.subAggregation(new WeightedAvgAggregationBuilder("subagg") .value(new MultiValuesSourceFieldConfig.Builder().setFieldName("num").build()) .weight(new MultiValuesSourceFieldConfig.Builder().setFieldName("num2").build()) @@ -529,10 +530,12 @@ public void testSearchWithParentJoin() throws IOException { client().performRequest(answerDoc2); client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh")); - TermsAggregationBuilder leafTermAgg = new TermsAggregationBuilder("top-names", ValueType.STRING) + TermsAggregationBuilder leafTermAgg = new TermsAggregationBuilder("top-names") + .userValueTypeHint(ValueType.STRING) .field("owner.display_name.keyword").size(10); ChildrenAggregationBuilder childrenAgg = new ChildrenAggregationBuilder("to-answers", "answer").subAggregation(leafTermAgg); - TermsAggregationBuilder termsAgg = new TermsAggregationBuilder("top-tags", ValueType.STRING).field("tags.keyword") + TermsAggregationBuilder termsAgg = new TermsAggregationBuilder("top-tags").userValueTypeHint(ValueType.STRING) + .field("tags.keyword") .size(10).subAggregation(childrenAgg); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(0).aggregation(termsAgg); @@ -744,15 +747,18 @@ public void testMultiSearch() throws Exception { public void testMultiSearch_withAgg() throws Exception { MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); SearchRequest searchRequest1 = new SearchRequest("index1"); - searchRequest1.source().size(0).aggregation(new TermsAggregationBuilder("name", ValueType.STRING).field("field.keyword") + searchRequest1.source().size(0).aggregation(new TermsAggregationBuilder("name").userValueTypeHint(ValueType.STRING) + .field("field.keyword") .order(BucketOrder.key(true))); multiSearchRequest.add(searchRequest1); SearchRequest searchRequest2 = new SearchRequest("index2"); - searchRequest2.source().size(0).aggregation(new TermsAggregationBuilder("name", ValueType.STRING).field("field.keyword") + searchRequest2.source().size(0).aggregation(new TermsAggregationBuilder("name").userValueTypeHint(ValueType.STRING) + .field("field.keyword") .order(BucketOrder.key(true))); multiSearchRequest.add(searchRequest2); SearchRequest searchRequest3 = new SearchRequest("index3"); - searchRequest3.source().size(0).aggregation(new TermsAggregationBuilder("name", ValueType.STRING).field("field.keyword") + searchRequest3.source().size(0).aggregation(new TermsAggregationBuilder("name").userValueTypeHint(ValueType.STRING) + .field("field.keyword") .order(BucketOrder.key(true))); multiSearchRequest.add(searchRequest3); diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java index e917d79d2d9f3..2c94c21c4f480 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java @@ -28,23 +28,19 @@ import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.ArrayValuesSourceAggregationBuilder; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; import java.util.Map; public class MatrixStatsAggregationBuilder - extends ArrayValuesSourceAggregationBuilder.LeafOnly { + extends ArrayValuesSourceAggregationBuilder.LeafOnly { public static final String NAME = "matrix_stats"; private MultiValueMode multiValueMode = MultiValueMode.AVG; public MatrixStatsAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); } protected MatrixStatsAggregationBuilder(MatrixStatsAggregationBuilder clone, @@ -62,7 +58,7 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu * Read from a stream. */ public MatrixStatsAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(in); } @Override @@ -81,7 +77,7 @@ public MultiValueMode multiValueMode() { @Override protected MatrixStatsAggregatorFactory innerBuild(QueryShardContext queryShardContext, - Map> configs, + Map configs, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException { return new MatrixStatsAggregatorFactory(name, configs, multiValueMode, queryShardContext, parent, subFactoriesBuilder, metaData); diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java index edd5f20d00928..6c044448a58ee 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -30,15 +31,16 @@ import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; -final class MatrixStatsAggregatorFactory extends ArrayValuesSourceAggregatorFactory { +final class MatrixStatsAggregatorFactory extends ArrayValuesSourceAggregatorFactory { private final MultiValueMode multiValueMode; MatrixStatsAggregatorFactory(String name, - Map> configs, + Map configs, MultiValueMode multiValueMode, QueryShardContext queryShardContext, AggregatorFactory parent, @@ -58,12 +60,21 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(Map valuesSources, + protected Aggregator doCreateInternal(Map valuesSources, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new MatrixStatsAggregator(name, valuesSources, searchContext, parent, multiValueMode, pipelineAggregators, metaData); + Map typedValuesSources = new HashMap<>(valuesSources.size()); + for (Map.Entry entry : valuesSources.entrySet()) { + if (entry.getValue() instanceof ValuesSource.Numeric == false) { + throw new AggregationExecutionException("ValuesSource type " + entry.getValue().toString() + + "is not supported for aggregation " + this.name()); + } + // TODO: There must be a better option than this. + typedValuesSources.put(entry.getKey(), (ValuesSource.Numeric) entry.getValue()); + } + return new MatrixStatsAggregator(name, typedValuesSources, searchContext, parent, multiValueMode, pipelineAggregators, metaData); } } diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java index 05b229b1219a6..b9939efc4e650 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java @@ -18,18 +18,11 @@ */ package org.elasticsearch.search.aggregations.support; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.script.Script; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.aggregations.AggregationInitializationException; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -44,19 +37,19 @@ import java.util.Map; import java.util.Objects; -public abstract class ArrayValuesSourceAggregationBuilder> +public abstract class ArrayValuesSourceAggregationBuilder> extends AbstractAggregationBuilder { public static final ParseField MULTIVALUE_MODE_FIELD = new ParseField("mode"); - public abstract static class LeafOnly> - extends ArrayValuesSourceAggregationBuilder { + public abstract static class LeafOnly> + extends ArrayValuesSourceAggregationBuilder { - protected LeafOnly(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) { - super(name, valuesSourceType, targetValueType); + protected LeafOnly(String name) { + super(name); } - protected LeafOnly(LeafOnly clone, Builder factoriesBuilder, Map metaData) { + protected LeafOnly(LeafOnly clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); if (factoriesBuilder.count() > 0) { throw new AggregationInitializationException("Aggregator [" + name + "] of type [" @@ -65,18 +58,10 @@ protected LeafOnly(LeafOnly clone, Builder factoriesBuilder, Map fields = Collections.emptyList(); - private ValueType valueType = null; + /* The parser doesn't support setting userValueTypeHint (aka valueType), but we do serialize and deserialize it, so keeping it around + for now so as to not break BWC. Future refactors should feel free to remove this field. --Tozzi 2020-01-16 + */ + private ValueType userValueTypeHint = null; private String format = null; private Object missing = null; private Map missingMap = Collections.emptyMap(); - protected ArrayValuesSourceAggregationBuilder(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) { + protected ArrayValuesSourceAggregationBuilder(String name) { super(name); - if (valuesSourceType == null) { - throw new IllegalArgumentException("[valuesSourceType] must not be null: [" + name + "]"); - } - this.valuesSourceType = valuesSourceType; - this.targetValueType = targetValueType; } - protected ArrayValuesSourceAggregationBuilder(ArrayValuesSourceAggregationBuilder clone, + protected ArrayValuesSourceAggregationBuilder(ArrayValuesSourceAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); - this.valuesSourceType = clone.valuesSourceType; - this.targetValueType = clone.targetValueType; this.fields = new ArrayList<>(clone.fields); - this.valueType = clone.valueType; + this.userValueTypeHint = clone.userValueTypeHint; this.format = clone.format; this.missingMap = new HashMap<>(clone.missingMap); this.missing = clone.missing; } - protected ArrayValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType) + protected ArrayValuesSourceAggregationBuilder(StreamInput in) throws IOException { super(in); - assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType"; - this.valuesSourceType = valuesSourceType; - this.targetValueType = targetValueType; - read(in); - } - - protected ArrayValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType) throws IOException { - super(in); - assert serializeTargetValueType() : "Wrong read constructor called for subclass that serializes its targetValueType"; - this.valuesSourceType = valuesSourceType; - this.targetValueType = in.readOptionalWriteable(ValueType::readFromStream); read(in); } @@ -138,18 +106,15 @@ protected ArrayValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType v @SuppressWarnings("unchecked") private void read(StreamInput in) throws IOException { fields = (ArrayList)in.readGenericValue(); - valueType = in.readOptionalWriteable(ValueType::readFromStream); + userValueTypeHint = in.readOptionalWriteable(ValueType::readFromStream); format = in.readOptionalString(); missingMap = in.readMap(); } @Override protected final void doWriteTo(StreamOutput out) throws IOException { - if (serializeTargetValueType()) { - out.writeOptionalWriteable(targetValueType); - } out.writeGenericValue(fields); - out.writeOptionalWriteable(valueType); + out.writeOptionalWriteable(userValueTypeHint); out.writeOptionalString(format); out.writeMap(missingMap); innerWriteTo(out); @@ -179,25 +144,6 @@ public List fields() { return fields; } - /** - * Sets the {@link ValueType} for the value produced by this aggregation - */ - @SuppressWarnings("unchecked") - public AB valueType(ValueType valueType) { - if (valueType == null) { - throw new IllegalArgumentException("[valueType] must not be null: [" + name + "]"); - } - this.valueType = valueType; - return (AB) this; - } - - /** - * Gets the {@link ValueType} for the value produced by this aggregation - */ - public ValueType valueType() { - return valueType; - } - /** * Sets the format to use for the output of the aggregation. */ @@ -239,96 +185,27 @@ public Map missingMap() { } @Override - protected final ArrayValuesSourceAggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { - Map> configs = resolveConfig(queryShardContext); - ArrayValuesSourceAggregatorFactory factory = innerBuild(queryShardContext, configs, parent, subFactoriesBuilder); + protected final ArrayValuesSourceAggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { + Map configs = resolveConfig(queryShardContext); + ArrayValuesSourceAggregatorFactory factory = innerBuild(queryShardContext, configs, parent, subFactoriesBuilder); return factory; } - protected Map> resolveConfig(QueryShardContext queryShardContext) { - HashMap> configs = new HashMap<>(); + protected Map resolveConfig(QueryShardContext queryShardContext) { + HashMap configs = new HashMap<>(); for (String field : fields) { - ValuesSourceConfig config = config(queryShardContext, field, null); + ValuesSourceConfig config = ValuesSourceConfig.resolveUnregistered(queryShardContext, userValueTypeHint, field, null, + missingMap.get(field), null, format, CoreValuesSourceType.BYTES); configs.put(field, config); } return configs; } - protected abstract ArrayValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - Map> configs, - AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder) throws IOException; - - public ValuesSourceConfig config(QueryShardContext queryShardContext, String field, Script script) { - - ValueType valueType = this.valueType != null ? this.valueType : targetValueType; - - if (field == null) { - if (script == null) { - ValuesSourceConfig config = new ValuesSourceConfig<>(CoreValuesSourceType.ANY); - return config.format(resolveFormat(null, valueType)); - } - ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType; - if (valuesSourceType == null || valuesSourceType == CoreValuesSourceType.ANY) { - // the specific value source type is undefined, but for scripts, - // we need to have a specific value source - // type to know how to handle the script values, so we fallback - // on Bytes - valuesSourceType = CoreValuesSourceType.BYTES; - } - ValuesSourceConfig config = new ValuesSourceConfig<>(valuesSourceType); - config.missing(missingMap.get(field)); - return config.format(resolveFormat(format, valueType)); - } - - MappedFieldType fieldType = queryShardContext.getMapperService().fieldType(field); - if (fieldType == null) { - ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType; - ValuesSourceConfig config = new ValuesSourceConfig<>(valuesSourceType); - config.missing(missingMap.get(field)); - config.format(resolveFormat(format, valueType)); - return config.unmapped(true); - } - - IndexFieldData indexFieldData = queryShardContext.getForField(fieldType); - - ValuesSourceConfig config; - if (valuesSourceType == CoreValuesSourceType.ANY) { - if (indexFieldData instanceof IndexNumericFieldData) { - config = new ValuesSourceConfig<>(CoreValuesSourceType.NUMERIC); - } else if (indexFieldData instanceof IndexGeoPointFieldData) { - config = new ValuesSourceConfig<>(CoreValuesSourceType.GEOPOINT); - } else { - config = new ValuesSourceConfig<>(CoreValuesSourceType.BYTES); - } - } else { - config = new ValuesSourceConfig<>(valuesSourceType); - } - - config.fieldContext(new FieldContext(field, indexFieldData, fieldType)); - config.missing(missingMap.get(field)); - return config.format(fieldType.docValueFormat(format, null)); - } - - private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValueType valueType) { - if (valueType == null) { - return DocValueFormat.RAW; // we can't figure it out - } - DocValueFormat valueFormat = valueType.defaultFormat(); - if (valueFormat instanceof DocValueFormat.Decimal && format != null) { - valueFormat = new DocValueFormat.Decimal(format); - } - return valueFormat; - } - - /** - * Should this builder serialize its targetValueType? Defaults to false. All subclasses that override this to true - * should use the three argument read constructor rather than the four argument version. - */ - protected boolean serializeTargetValueType() { - return false; - } + protected abstract ArrayValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + Map configs, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder) throws IOException; @Override public final XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException { @@ -343,8 +220,8 @@ public final XContentBuilder internalXContent(XContentBuilder builder, Params pa if (format != null) { builder.field(CommonFields.FORMAT.getPreferredName(), format); } - if (valueType != null) { - builder.field(CommonFields.VALUE_TYPE.getPreferredName(), valueType.getPreferredName()); + if (userValueTypeHint != null) { + builder.field(CommonFields.VALUE_TYPE.getPreferredName(), userValueTypeHint.getPreferredName()); } doXContentBody(builder, params); builder.endObject(); @@ -355,7 +232,7 @@ public final XContentBuilder internalXContent(XContentBuilder builder, Params pa @Override public int hashCode() { - return Objects.hash(super.hashCode(), fields, format, missing, targetValueType, valueType, valuesSourceType); + return Objects.hash(super.hashCode(), fields, format, missing, userValueTypeHint); } @Override @@ -363,12 +240,10 @@ public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; if (super.equals(obj) == false) return false; - ArrayValuesSourceAggregationBuilder other = (ArrayValuesSourceAggregationBuilder) obj; + ArrayValuesSourceAggregationBuilder other = (ArrayValuesSourceAggregationBuilder) obj; return Objects.equals(fields, other.fields) && Objects.equals(format, other.format) && Objects.equals(missing, other.missing) - && Objects.equals(targetValueType, other.targetValueType) - && Objects.equals(valueType, other.valueType) - && Objects.equals(valuesSourceType, other.valuesSourceType); + && Objects.equals(userValueTypeHint, other.userValueTypeHint); } } diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java index a21c4dbd36660..d1124c9e693a5 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java @@ -31,12 +31,12 @@ import java.util.List; import java.util.Map; -public abstract class ArrayValuesSourceAggregatorFactory +public abstract class ArrayValuesSourceAggregatorFactory extends AggregatorFactory { - protected Map> configs; + protected Map configs; - public ArrayValuesSourceAggregatorFactory(String name, Map> configs, + public ArrayValuesSourceAggregatorFactory(String name, Map configs, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { @@ -50,10 +50,10 @@ public Aggregator createInternal(SearchContext searchContext, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - HashMap valuesSources = new HashMap<>(); + HashMap valuesSources = new HashMap<>(); - for (Map.Entry> config : configs.entrySet()) { - VS vs = config.getValue().toValuesSource(queryShardContext); + for (Map.Entry config : configs.entrySet()) { + ValuesSource vs = config.getValue().toValuesSource(); if (vs != null) { valuesSources.put(config.getKey(), vs); } @@ -70,7 +70,7 @@ protected abstract Aggregator createUnmapped(SearchContext searchContext, List pipelineAggregators, Map metaData) throws IOException; - protected abstract Aggregator doCreateInternal(Map valuesSources, + protected abstract Aggregator doCreateInternal(Map valuesSources, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java index a0e7016291b17..4d935c92c2e1f 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java @@ -35,13 +35,6 @@ public abstract class ArrayValuesSourceParser implements Aggregator.Parser { - public abstract static class AnyValuesSourceParser extends ArrayValuesSourceParser { - - protected AnyValuesSourceParser(boolean formattable) { - super(formattable, CoreValuesSourceType.ANY, null); - } - } - public abstract static class NumericValuesSourceParser extends ArrayValuesSourceParser { protected NumericValuesSourceParser(boolean formattable) { @@ -74,7 +67,7 @@ private ArrayValuesSourceParser(boolean formattable, ValuesSourceType valuesSour } @Override - public final ArrayValuesSourceAggregationBuilder parse(String aggregationName, XContentParser parser) + public final ArrayValuesSourceAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException { List fields = null; @@ -139,7 +132,7 @@ private ArrayValuesSourceParser(boolean formattable, ValuesSourceType valuesSour } } - ArrayValuesSourceAggregationBuilder factory = createFactory(aggregationName, this.valuesSourceType, this.targetValueType, + ArrayValuesSourceAggregationBuilder factory = createFactory(aggregationName, this.valuesSourceType, this.targetValueType, otherOptions); if (fields != null) { factory.fields(fields); @@ -193,10 +186,10 @@ private void parseMissingAndAdd(final String aggregationName, final String curre * method * @return the created factory */ - protected abstract ArrayValuesSourceAggregationBuilder createFactory(String aggregationName, - ValuesSourceType valuesSourceType, - ValueType targetValueType, - Map otherOptions); + protected abstract ArrayValuesSourceAggregationBuilder createFactory(String aggregationName, + ValuesSourceType valuesSourceType, + ValueType targetValueType, + Map otherOptions); /** * Allows subclasses of {@link ArrayValuesSourceParser} to parse extra diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 81c092f3a2de6..c969ee00d9fae 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -60,6 +60,8 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java index 23a89752964b2..8eedf429fa5d7 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.join.mapper.ParentIdFieldMapper; @@ -34,19 +33,17 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.FieldContext; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; import java.util.Objects; public class ChildrenAggregationBuilder - extends ValuesSourceAggregationBuilder { + extends ValuesSourceAggregationBuilder { public static final String NAME = "children"; @@ -61,7 +58,7 @@ public class ChildrenAggregationBuilder * the type of children documents */ public ChildrenAggregationBuilder(String name, String childType) { - super(name, CoreValuesSourceType.BYTES, ValueType.STRING); + super(name); if (childType == null) { throw new IllegalArgumentException("[childType] must not be null: [" + name + "]"); } @@ -76,6 +73,11 @@ protected ChildrenAggregationBuilder(ChildrenAggregationBuilder clone, this.parentFilter = clone.parentFilter; } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new ChildrenAggregationBuilder(this, factoriesBuilder, metaData); @@ -85,7 +87,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return new ChildrenAggregatorFactory(name, config, childFilter, parentFilter, queryShardContext, parent, subFactoriesBuilder, metaData); } @Override - protected ValuesSourceConfig resolveConfig(QueryShardContext queryShardContext) { - ValuesSourceConfig config = new ValuesSourceConfig<>(CoreValuesSourceType.BYTES); - joinFieldResolveConfig(queryShardContext, config); - return config; - } - - private void joinFieldResolveConfig(QueryShardContext queryShardContext, ValuesSourceConfig config) { + protected ValuesSourceConfig resolveConfig(QueryShardContext queryShardContext) { + ValuesSourceConfig config; ParentJoinFieldMapper parentJoinFieldMapper = ParentJoinFieldMapper.getMapper(queryShardContext.getMapperService()); ParentIdFieldMapper parentIdFieldMapper = parentJoinFieldMapper.getParentIdFieldMapper(childType, false); if (parentIdFieldMapper != null) { parentFilter = parentIdFieldMapper.getParentFilter(); childFilter = parentIdFieldMapper.getChildFilter(childType); MappedFieldType fieldType = parentIdFieldMapper.fieldType(); - final SortedSetDVOrdinalsIndexFieldData fieldData = queryShardContext.getForField(fieldType); - config.fieldContext(new FieldContext(fieldType.name(), fieldData, fieldType)); + config = ValuesSourceConfig.resolveFieldOnly(fieldType, queryShardContext); } else { - config.unmapped(true); + // Unmapped field case + config = ValuesSourceConfig.resolveUnmapped(defaultValueSourceType(), queryShardContext); } + return config; } @Override diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java index 57a746677f881..f0c18e4eb46e5 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java @@ -21,12 +21,14 @@ import org.apache.lucene.search.Query; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.NonCollectingAggregator; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; @@ -36,13 +38,13 @@ import java.util.List; import java.util.Map; -public class ChildrenAggregatorFactory extends ValuesSourceAggregatorFactory { +public class ChildrenAggregatorFactory extends ValuesSourceAggregatorFactory { private final Query parentFilter; private final Query childFilter; public ChildrenAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, Query childFilter, Query parentFilter, QueryShardContext context, @@ -67,12 +69,17 @@ public InternalAggregation buildEmptyAggregation() { } @Override - protected Aggregator doCreateInternal(WithOrdinals valuesSource, + protected Aggregator doCreateInternal(ValuesSource rawValuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { + if (rawValuesSource instanceof WithOrdinals == false) { + throw new AggregationExecutionException("ValuesSource type " + rawValuesSource.toString() + + "is not supported for aggregation " + this.name()); + } + WithOrdinals valuesSource = (WithOrdinals) rawValuesSource; long maxOrd = valuesSource.globalMaxOrd(searchContext.searcher()); if (collectsFromSingleBucket) { return new ParentToChildrenAggregator(name, factories, searchContext, parent, childFilter, diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java index dd58d5b11e55a..f920965e178cd 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.join.mapper.ParentIdFieldMapper; @@ -34,19 +33,17 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.FieldContext; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; import java.util.Objects; public class ParentAggregationBuilder - extends ValuesSourceAggregationBuilder { + extends ValuesSourceAggregationBuilder { public static final String NAME = "parent"; @@ -61,7 +58,7 @@ public class ParentAggregationBuilder * the type of children documents */ public ParentAggregationBuilder(String name, String childType) { - super(name, CoreValuesSourceType.BYTES, ValueType.STRING); + super(name); if (childType == null) { throw new IllegalArgumentException("[childType] must not be null: [" + name + "]"); } @@ -76,6 +73,11 @@ protected ParentAggregationBuilder(ParentAggregationBuilder clone, this.parentFilter = clone.parentFilter; } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new ParentAggregationBuilder(this, factoriesBuilder, metaData); @@ -85,7 +87,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return new ParentAggregatorFactory(name, config, childFilter, parentFilter, queryShardContext, parent, subFactoriesBuilder, metaData); } @Override - protected ValuesSourceConfig resolveConfig(QueryShardContext queryShardContext) { - ValuesSourceConfig config = new ValuesSourceConfig<>(CoreValuesSourceType.BYTES); - joinFieldResolveConfig(queryShardContext, config); - return config; - } - - private void joinFieldResolveConfig(QueryShardContext queryShardContext, ValuesSourceConfig config) { + protected ValuesSourceConfig resolveConfig(QueryShardContext queryShardContext) { + ValuesSourceConfig config; ParentJoinFieldMapper parentJoinFieldMapper = ParentJoinFieldMapper.getMapper(queryShardContext.getMapperService()); ParentIdFieldMapper parentIdFieldMapper = parentJoinFieldMapper.getParentIdFieldMapper(childType, false); if (parentIdFieldMapper != null) { parentFilter = parentIdFieldMapper.getParentFilter(); childFilter = parentIdFieldMapper.getChildFilter(childType); MappedFieldType fieldType = parentIdFieldMapper.fieldType(); - final SortedSetDVOrdinalsIndexFieldData fieldData = queryShardContext.getForField(fieldType); - config.fieldContext(new FieldContext(fieldType.name(), fieldData, fieldType)); + config = ValuesSourceConfig.resolveFieldOnly(fieldType, queryShardContext); } else { - config.unmapped(true); + // unmapped case + config = ValuesSourceConfig.resolveUnmapped(defaultValueSourceType(), queryShardContext); } + return config; } @Override diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java index dc64d0308cd09..a7a2730f41435 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java @@ -21,12 +21,14 @@ import org.apache.lucene.search.Query; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.NonCollectingAggregator; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; @@ -36,13 +38,13 @@ import java.util.List; import java.util.Map; -public class ParentAggregatorFactory extends ValuesSourceAggregatorFactory { +public class ParentAggregatorFactory extends ValuesSourceAggregatorFactory { private final Query parentFilter; private final Query childFilter; public ParentAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, Query childFilter, Query parentFilter, QueryShardContext queryShardContext, @@ -67,12 +69,17 @@ public InternalAggregation buildEmptyAggregation() { } @Override - protected Aggregator doCreateInternal(WithOrdinals valuesSource, + protected Aggregator doCreateInternal(ValuesSource rawValuesSource, SearchContext searchContext, Aggregator children, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { + if (rawValuesSource instanceof WithOrdinals == false) { + throw new AggregationExecutionException("ValuesSource type " + rawValuesSource.toString() + + "is not supported for aggregation " + this.name()); + } + WithOrdinals valuesSource = (WithOrdinals) rawValuesSource; long maxOrd = valuesSource.globalMaxOrd(searchContext.searcher()); if (collectsFromSingleBucket) { return new ChildrenToParentAggregator(name, factories, searchContext, children, childFilter, diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java index a161e2a6a13e6..cd7f39508c3ac 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java @@ -304,7 +304,8 @@ private void testCaseTerms(Query query, IndexSearcher indexSearcher, Consumer verify) throws IOException { AggregationBuilder aggregationBuilder = - new TermsAggregationBuilder("subvalue_terms", ValueType.LONG).field("subNumber"). + new TermsAggregationBuilder("subvalue_terms").userValueTypeHint(ValueType.LONG).field("subNumber"). subAggregation(new ParentAggregationBuilder("to_parent", CHILD_TYPE). - subAggregation(new TermsAggregationBuilder("value_terms", ValueType.LONG).field("number"))); + subAggregation(new TermsAggregationBuilder("value_terms").userValueTypeHint(ValueType.LONG).field("number"))); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); fieldType.setName("number"); diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java index d2f876cd17848..0c16e539fcaaf 100644 --- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java +++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java @@ -524,31 +524,32 @@ public void testTermsAggsWithProfile() throws Exception { private static SearchSourceBuilder buildTermsAggsSource() { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.size(0); - TermsAggregationBuilder cluster = new TermsAggregationBuilder("cluster123", ValueType.STRING); + TermsAggregationBuilder cluster = new TermsAggregationBuilder("cluster123").userValueTypeHint(ValueType.STRING); cluster.field("_index"); - TermsAggregationBuilder type = new TermsAggregationBuilder("type", ValueType.STRING); + TermsAggregationBuilder type = new TermsAggregationBuilder("type").userValueTypeHint(ValueType.STRING); type.field("type.keyword"); type.showTermDocCountError(true); type.order(BucketOrder.key(true)); cluster.subAggregation(type); sourceBuilder.aggregation(cluster); - TermsAggregationBuilder tags = new TermsAggregationBuilder("tags", ValueType.STRING); + TermsAggregationBuilder tags = new TermsAggregationBuilder("tags").userValueTypeHint(ValueType.STRING); tags.field("tags.keyword"); tags.showTermDocCountError(true); tags.size(100); sourceBuilder.aggregation(tags); - TermsAggregationBuilder tags2 = new TermsAggregationBuilder("tags", ValueType.STRING); + TermsAggregationBuilder tags2 = new TermsAggregationBuilder("tags").userValueTypeHint(ValueType.STRING); tags2.field("tags.keyword"); tags.subAggregation(tags2); FilterAggregationBuilder answers = new FilterAggregationBuilder("answers", new TermQueryBuilder("type", "answer")); - TermsAggregationBuilder answerPerQuestion = new TermsAggregationBuilder("answer_per_question", ValueType.STRING); + TermsAggregationBuilder answerPerQuestion = new TermsAggregationBuilder("answer_per_question") + .userValueTypeHint(ValueType.STRING); answerPerQuestion.showTermDocCountError(true); answerPerQuestion.field("questionId.keyword"); answers.subAggregation(answerPerQuestion); - TermsAggregationBuilder answerPerUser = new TermsAggregationBuilder("answer_per_user", ValueType.STRING); + TermsAggregationBuilder answerPerUser = new TermsAggregationBuilder("answer_per_user").userValueTypeHint(ValueType.STRING); answerPerUser.field("user.keyword"); answerPerUser.size(30); answerPerUser.showTermDocCountError(true); @@ -563,7 +564,7 @@ public void testDateHistogram() throws Exception { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.size(0); searchRequest.source(sourceBuilder); - TermsAggregationBuilder tags = new TermsAggregationBuilder("tags", ValueType.STRING); + TermsAggregationBuilder tags = new TermsAggregationBuilder("tags").userValueTypeHint(ValueType.STRING); tags.field("tags.keyword"); tags.showTermDocCountError(true); DateHistogramAggregationBuilder creation = new DateHistogramAggregationBuilder("creation"); @@ -580,7 +581,7 @@ public void testCardinalityAgg() throws Exception { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.size(0); searchRequest.source(sourceBuilder); - CardinalityAggregationBuilder tags = new CardinalityAggregationBuilder("tags", ValueType.STRING); + CardinalityAggregationBuilder tags = new CardinalityAggregationBuilder("tags").userValueTypeHint(ValueType.STRING); tags.field("tags.keyword"); sourceBuilder.aggregation(tags); duelSearch(searchRequest, CCSDuelIT::assertAggs); @@ -619,7 +620,7 @@ public void testTopHits() throws Exception { topHits.size(10); topHits.sort("creationDate", SortOrder.DESC); topHits.sort("id", SortOrder.ASC); - TermsAggregationBuilder tags = new TermsAggregationBuilder("tags", ValueType.STRING); + TermsAggregationBuilder tags = new TermsAggregationBuilder("tags").userValueTypeHint(ValueType.STRING); tags.field("tags.keyword"); tags.size(10); tags.subAggregation(topHits); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_rare_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_rare_terms.yml index 6d1e2d606a4b2..35ed9cc0c4b7b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_rare_terms.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_rare_terms.yml @@ -112,7 +112,7 @@ setup: - length: { aggregations.ip_terms.buckets: 0 } - do: - catch: request + catch: /Aggregation \[ip_terms\] cannot support regular expression style include\/exclude settings as they can only be applied to string fields\. Use an array of values for include\/exclude clauses/ search: index: test_1 body: { "size" : 0, "aggs" : { "ip_terms" : { "rare_terms" : { "field" : "ip", "exclude" : "127.*" } } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml index dabd4aebf815d..d7759afe4a907 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml @@ -133,7 +133,7 @@ - length: { aggregations.ip_terms.buckets: 0 } - do: - catch: request + catch: /Aggregation \[ip_terms\] cannot support regular expression style include\/exclude settings as they can only be applied to string fields\. Use an array of values for include\/exclude clauses/ search: rest_total_hits_as_int: true body: { "size" : 0, "aggs" : { "ip_terms" : { "significant_terms" : { "field" : "ip", "exclude" : "127.*" } } } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 16b7c069bc5cd..30bd74e3edd2c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -61,6 +61,7 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.plugins.IndexStorePlugin; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -388,23 +389,22 @@ public static Type defaultStoreType(final boolean allowMmap) { } } - public IndexService newIndexService( - IndexService.IndexCreationContext indexCreationContext, - NodeEnvironment environment, - NamedXContentRegistry xContentRegistry, - IndexService.ShardStoreDeleter shardStoreDeleter, - CircuitBreakerService circuitBreakerService, - BigArrays bigArrays, - ThreadPool threadPool, - ScriptService scriptService, - ClusterService clusterService, - Client client, - IndicesQueryCache indicesQueryCache, - MapperRegistry mapperRegistry, - IndicesFieldDataCache indicesFieldDataCache, - NamedWriteableRegistry namedWriteableRegistry, - BooleanSupplier idFieldDataEnabled) - throws IOException { + public IndexService newIndexService(IndexService.IndexCreationContext indexCreationContext, + NodeEnvironment environment, + NamedXContentRegistry xContentRegistry, + IndexService.ShardStoreDeleter shardStoreDeleter, + CircuitBreakerService circuitBreakerService, + BigArrays bigArrays, + ThreadPool threadPool, + ScriptService scriptService, + ClusterService clusterService, + Client client, + IndicesQueryCache indicesQueryCache, + MapperRegistry mapperRegistry, + IndicesFieldDataCache indicesFieldDataCache, + NamedWriteableRegistry namedWriteableRegistry, + BooleanSupplier idFieldDataEnabled, + ValuesSourceRegistry valuesSourceRegistry) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper.get() == null ? (shard) -> null : indexReaderWrapper.get(); @@ -431,7 +431,8 @@ public IndexService newIndexService( new SimilarityService(indexSettings, scriptService, similarities), shardStoreDeleter, indexAnalyzers, engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, client, queryCache, directoryFactory, eventListener, readerWrapperFactory, mapperRegistry, indicesFieldDataCache, searchOperationListeners, - indexOperationListeners, namedWriteableRegistry, idFieldDataEnabled, allowExpensiveQueries, expressionResolver); + indexOperationListeners, namedWriteableRegistry, idFieldDataEnabled, allowExpensiveQueries, expressionResolver, + valuesSourceRegistry); success = true; return indexService; } finally { diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index fdd28a9861600..502f89ee18d4e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -81,6 +81,7 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.plugins.IndexStorePlugin; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.threadpool.ThreadPool; import java.io.Closeable; @@ -145,6 +146,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final CircuitBreakerService circuitBreakerService; private final IndexNameExpressionResolver expressionResolver; private Supplier indexSortSupplier; + private ValuesSourceRegistry valuesSourceRegistry; public IndexService( IndexSettings indexSettings, @@ -171,7 +173,8 @@ public IndexService( NamedWriteableRegistry namedWriteableRegistry, BooleanSupplier idFieldDataEnabled, BooleanSupplier allowExpensiveQueries, - IndexNameExpressionResolver expressionResolver) { + IndexNameExpressionResolver expressionResolver, + ValuesSourceRegistry valuesSourceRegistry) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; this.indexSettings = indexSettings; @@ -180,6 +183,7 @@ public IndexService( this.namedWriteableRegistry = namedWriteableRegistry; this.circuitBreakerService = circuitBreakerService; this.expressionResolver = expressionResolver; + this.valuesSourceRegistry = valuesSourceRegistry; if (needsMapperService(indexSettings, indexCreationContext)) { assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, @@ -576,7 +580,7 @@ public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searche return new QueryShardContext( shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries); + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 009848df1d8a9..3def7265d4c9c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -193,7 +193,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public ValuesSourceType getValuesSourceType() { - return CoreValuesSourceType.NUMERIC; + return CoreValuesSourceType.BOOLEAN; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 4e6b04a72418a..def230e2a5c62 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -538,7 +538,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public ValuesSourceType getValuesSourceType() { - return CoreValuesSourceType.NUMERIC; + return CoreValuesSourceType.DATE; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 789147775fe5a..39505d61d586e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -48,6 +48,8 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index c1cf50a56cde5..a3abf963104ae 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -298,7 +298,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public ValuesSourceType getValuesSourceType() { - return CoreValuesSourceType.BYTES; + return CoreValuesSourceType.IP; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 611a6fbe6d509..98db6e5dfb8a0 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -56,6 +56,7 @@ import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.transport.RemoteClusterAware; @@ -99,6 +100,7 @@ public class QueryShardContext extends QueryRewriteContext { private boolean allowUnmappedFields; private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; + private ValuesSourceRegistry valuesSourceRegistry; public QueryShardContext(int shardId, IndexSettings indexSettings, @@ -115,18 +117,19 @@ public QueryShardContext(int shardId, LongSupplier nowInMillis, String clusterAlias, Predicate indexNameMatcher, - BooleanSupplier allowExpensiveQueries) { + BooleanSupplier allowExpensiveQueries, + ValuesSourceRegistry valuesSourceRegistry) { this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID()), allowExpensiveQueries); + indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry); } public QueryShardContext(QueryShardContext source) { this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries); + source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry); } private QueryShardContext(int shardId, @@ -144,7 +147,8 @@ private QueryShardContext(int shardId, LongSupplier nowInMillis, Predicate indexNameMatcher, Index fullyQualifiedIndex, - BooleanSupplier allowExpensiveQueries) { + BooleanSupplier allowExpensiveQueries, + ValuesSourceRegistry valuesSourceRegistry) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; this.similarityService = similarityService; @@ -160,6 +164,7 @@ private QueryShardContext(int shardId, this.indexNameMatcher = indexNameMatcher; this.fullyQualifiedIndex = fullyQualifiedIndex; this.allowExpensiveQueries = allowExpensiveQueries; + this.valuesSourceRegistry = valuesSourceRegistry; } private void reset() { @@ -258,6 +263,10 @@ public Analyzer getSearchQuoteAnalyzer(MappedFieldType fieldType) { return getMapperService().searchQuoteAnalyzer(); } + public ValuesSourceRegistry getValuesSourceRegistry() { + return valuesSourceRegistry; + } + public void setAllowUnmappedFields(boolean allowUnmappedFields) { this.allowUnmappedFields = allowUnmappedFields; } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index a1cf0ce39cf98..964609ebfd74b 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -126,6 +126,7 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchRequest; @@ -227,6 +228,7 @@ public class IndicesService extends AbstractLifecycleComponent private final EsThreadPoolExecutor danglingIndicesThreadPoolExecutor; private final Set danglingIndicesToWrite = Sets.newConcurrentHashSet(); private final boolean nodeWriteDanglingIndicesInfo; + private ValuesSourceRegistry valuesSourceRegistry; @Override @@ -241,12 +243,13 @@ public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvi IndexScopedSettings indexScopedSettings, CircuitBreakerService circuitBreakerService, BigArrays bigArrays, ScriptService scriptService, ClusterService clusterService, Client client, MetaStateService metaStateService, Collection>> engineFactoryProviders, - Map directoryFactories) { + Map directoryFactories, ValuesSourceRegistry valuesSourceRegistry) { this.settings = settings; this.threadPool = threadPool; this.pluginsService = pluginsService; this.nodeEnv = nodeEnv; this.xContentRegistry = xContentRegistry; + this.valuesSourceRegistry = valuesSourceRegistry; this.shardsClosedTimeout = settings.getAsTime(INDICES_SHARDS_CLOSED_TIMEOUT, new TimeValue(1, TimeUnit.DAYS)); this.analysisRegistry = analysisRegistry; this.indexNameExpressionResolver = indexNameExpressionResolver; @@ -615,7 +618,8 @@ private synchronized IndexService createIndexService(IndexService.IndexCreationC mapperRegistry, indicesFieldDataCache, namedWriteableRegistry, - this::isIdFieldDataEnabled + this::isIdFieldDataEnabled, + valuesSourceRegistry ); } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 63a756887d864..5e80f71429d4c 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -454,7 +454,8 @@ protected Node(final Environment initialEnvironment, new IndicesService(settings, pluginsService, nodeEnvironment, xContentRegistry, analysisModule.getAnalysisRegistry(), clusterModule.getIndexNameExpressionResolver(), indicesModule.getMapperRegistry(), namedWriteableRegistry, threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, bigArrays, scriptService, - clusterService, client, metaStateService, engineFactoryProviders, indexStoreFactories); + clusterService, client, metaStateService, engineFactoryProviders, indexStoreFactories, + searchModule.getValuesSourceRegistry()); final AliasValidator aliasValidator = new AliasValidator(); diff --git a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java index e67fd5f143a1d..d2f2aec3c76dd 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java @@ -42,6 +42,7 @@ import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; import org.elasticsearch.search.rescore.Rescorer; @@ -55,6 +56,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.function.BiFunction; +import java.util.function.Consumer; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -112,6 +114,13 @@ default List> getQueries() { default List getAggregations() { return emptyList(); } + /** + * Allows plugins to register new aggregations using aggregation names that are already defined + * in Core, as long as the new aggregations target different ValuesSourceTypes + */ + default List> getBareAggregatorRegistrar() { + return emptyList(); + } /** * The new {@link PipelineAggregator}s added by this plugin. */ @@ -251,6 +260,7 @@ public QuerySpec(String name, Writeable.Reader reader, QueryParser parser) */ class AggregationSpec extends SearchExtensionSpec> { private final Map> resultReaders = new TreeMap<>(); + private Consumer aggregatorRegistrar; /** * Specification for an {@link Aggregation}. @@ -333,6 +343,23 @@ public AggregationSpec addResultReader(String writeableName, Writeable.Reader> getResultReaders() { return resultReaders; } + + /** + * Get the function to register the {@link org.elasticsearch.search.aggregations.support.ValuesSource} to aggregator mappings for + * this aggregation + */ + public Consumer getAggregatorRegistrar() { + return aggregatorRegistrar; + } + + /** + * Set the function to register the {@link org.elasticsearch.search.aggregations.support.ValuesSource} to aggregator mappings for + * this aggregation + */ + public AggregationSpec setAggregatorRegistrar(Consumer aggregatorRegistrar) { + this.aggregatorRegistrar = aggregatorRegistrar; + return this; + } } /** diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index dea00a9b3012d..6b2b0d846e132 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -228,6 +228,7 @@ import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.SumBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.SumBucketPipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.subphase.FetchDocValuesPhase; @@ -287,6 +288,7 @@ public class SearchModule { private final Settings settings; private final List namedWriteables = new ArrayList<>(); private final List namedXContents = new ArrayList<>(); + private ValuesSourceRegistry valuesSourceRegistry; /** * Constructs a new SearchModule object @@ -298,6 +300,7 @@ public class SearchModule { */ public SearchModule(Settings settings, List plugins) { this.settings = settings; + this.valuesSourceRegistry = new ValuesSourceRegistry(); registerSuggesters(plugins); highlighters = setupHighlighters(settings, plugins); registerScoreFunctions(plugins); @@ -323,6 +326,10 @@ public List getNamedXContents() { return namedXContents; } + public ValuesSourceRegistry getValuesSourceRegistry() { + return valuesSourceRegistry; + } + /** * Returns the {@link Highlighter} registry */ @@ -332,34 +339,45 @@ public Map getHighlighters() { private void registerAggregations(List plugins) { registerAggregation(new AggregationSpec(AvgAggregationBuilder.NAME, AvgAggregationBuilder::new, AvgAggregationBuilder.PARSER) - .addResultReader(InternalAvg::new)); + .addResultReader(InternalAvg::new) + .setAggregatorRegistrar(AvgAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(WeightedAvgAggregationBuilder.NAME, WeightedAvgAggregationBuilder::new, WeightedAvgAggregationBuilder.PARSER).addResultReader(InternalWeightedAvg::new)); registerAggregation(new AggregationSpec(SumAggregationBuilder.NAME, SumAggregationBuilder::new, SumAggregationBuilder.PARSER) - .addResultReader(InternalSum::new)); + .addResultReader(InternalSum::new) + .setAggregatorRegistrar(SumAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(MinAggregationBuilder.NAME, MinAggregationBuilder::new, MinAggregationBuilder.PARSER) - .addResultReader(InternalMin::new)); + .addResultReader(InternalMin::new) + .setAggregatorRegistrar(MinAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(MaxAggregationBuilder.NAME, MaxAggregationBuilder::new, MaxAggregationBuilder.PARSER) - .addResultReader(InternalMax::new)); + .addResultReader(InternalMax::new) + .setAggregatorRegistrar(MaxAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(StatsAggregationBuilder.NAME, StatsAggregationBuilder::new, StatsAggregationBuilder.PARSER) - .addResultReader(InternalStats::new)); + .addResultReader(InternalStats::new) + .setAggregatorRegistrar(StatsAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(ExtendedStatsAggregationBuilder.NAME, ExtendedStatsAggregationBuilder::new, ExtendedStatsAggregationBuilder.PARSER).addResultReader(InternalExtendedStats::new)); registerAggregation(new AggregationSpec(ValueCountAggregationBuilder.NAME, ValueCountAggregationBuilder::new, - ValueCountAggregationBuilder.PARSER).addResultReader(InternalValueCount::new)); + ValueCountAggregationBuilder.PARSER) + .addResultReader(InternalValueCount::new) + .setAggregatorRegistrar(ValueCountAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(PercentilesAggregationBuilder.NAME, PercentilesAggregationBuilder::new, PercentilesAggregationBuilder.PARSER) .addResultReader(InternalTDigestPercentiles.NAME, InternalTDigestPercentiles::new) - .addResultReader(InternalHDRPercentiles.NAME, InternalHDRPercentiles::new)); + .addResultReader(InternalHDRPercentiles.NAME, InternalHDRPercentiles::new) + .setAggregatorRegistrar(PercentilesAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(PercentileRanksAggregationBuilder.NAME, PercentileRanksAggregationBuilder::new, PercentileRanksAggregationBuilder.PARSER) .addResultReader(InternalTDigestPercentileRanks.NAME, InternalTDigestPercentileRanks::new) - .addResultReader(InternalHDRPercentileRanks.NAME, InternalHDRPercentileRanks::new)); + .addResultReader(InternalHDRPercentileRanks.NAME, InternalHDRPercentileRanks::new) + .setAggregatorRegistrar(PercentileRanksAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(MedianAbsoluteDeviationAggregationBuilder.NAME, - MedianAbsoluteDeviationAggregationBuilder::new, MedianAbsoluteDeviationAggregationBuilder.PARSER) - .addResultReader(InternalMedianAbsoluteDeviation::new)); + MedianAbsoluteDeviationAggregationBuilder::new, MedianAbsoluteDeviationAggregationBuilder.PARSER) + .addResultReader(InternalMedianAbsoluteDeviation::new) + .setAggregatorRegistrar(MedianAbsoluteDeviationAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(CardinalityAggregationBuilder.NAME, CardinalityAggregationBuilder::new, - CardinalityAggregationBuilder.PARSER).addResultReader(InternalCardinality::new)); + CardinalityAggregationBuilder.PARSER).addResultReader(InternalCardinality::new) + .setAggregatorRegistrar(CardinalityAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(GlobalAggregationBuilder.NAME, GlobalAggregationBuilder::new, GlobalAggregationBuilder::parse).addResultReader(InternalGlobal::new)); registerAggregation(new AggregationSpec(MissingAggregationBuilder.NAME, MissingAggregationBuilder::new, @@ -382,37 +400,52 @@ private void registerAggregations(List plugins) { .addResultReader(StringTerms.NAME, StringTerms::new) .addResultReader(UnmappedTerms.NAME, UnmappedTerms::new) .addResultReader(LongTerms.NAME, LongTerms::new) - .addResultReader(DoubleTerms.NAME, DoubleTerms::new)); + .addResultReader(DoubleTerms.NAME, DoubleTerms::new) + .setAggregatorRegistrar(TermsAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(RareTermsAggregationBuilder.NAME, RareTermsAggregationBuilder::new, RareTermsAggregationBuilder.PARSER) .addResultReader(StringRareTerms.NAME, StringRareTerms::new) .addResultReader(UnmappedRareTerms.NAME, UnmappedRareTerms::new) - .addResultReader(LongRareTerms.NAME, LongRareTerms::new)); + .addResultReader(LongRareTerms.NAME, LongRareTerms::new) + .setAggregatorRegistrar(RareTermsAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(SignificantTermsAggregationBuilder.NAME, SignificantTermsAggregationBuilder::new, SignificantTermsAggregationBuilder::parse) .addResultReader(SignificantStringTerms.NAME, SignificantStringTerms::new) .addResultReader(SignificantLongTerms.NAME, SignificantLongTerms::new) - .addResultReader(UnmappedSignificantTerms.NAME, UnmappedSignificantTerms::new)); + .addResultReader(UnmappedSignificantTerms.NAME, UnmappedSignificantTerms::new) + .setAggregatorRegistrar(SignificantTermsAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(SignificantTextAggregationBuilder.NAME, SignificantTextAggregationBuilder::new, SignificantTextAggregationBuilder::parse)); registerAggregation(new AggregationSpec(RangeAggregationBuilder.NAME, RangeAggregationBuilder::new, - RangeAggregationBuilder.PARSER).addResultReader(InternalRange::new)); + RangeAggregationBuilder.PARSER) + .addResultReader(InternalRange::new) + .setAggregatorRegistrar(RangeAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(DateRangeAggregationBuilder.NAME, DateRangeAggregationBuilder::new, - DateRangeAggregationBuilder.PARSER).addResultReader(InternalDateRange::new)); + DateRangeAggregationBuilder.PARSER) + .addResultReader(InternalDateRange::new) + .setAggregatorRegistrar(DateRangeAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(IpRangeAggregationBuilder.NAME, IpRangeAggregationBuilder::new, IpRangeAggregationBuilder.PARSER).addResultReader(InternalBinaryRange::new)); registerAggregation(new AggregationSpec(HistogramAggregationBuilder.NAME, HistogramAggregationBuilder::new, - HistogramAggregationBuilder.PARSER).addResultReader(InternalHistogram::new)); + HistogramAggregationBuilder.PARSER) + .addResultReader(InternalHistogram::new) + .setAggregatorRegistrar(HistogramAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(DateHistogramAggregationBuilder.NAME, DateHistogramAggregationBuilder::new, - DateHistogramAggregationBuilder.PARSER).addResultReader(InternalDateHistogram::new)); + DateHistogramAggregationBuilder.PARSER) + .addResultReader(InternalDateHistogram::new) + .setAggregatorRegistrar(DateHistogramAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(AutoDateHistogramAggregationBuilder.NAME, AutoDateHistogramAggregationBuilder::new, AutoDateHistogramAggregationBuilder.PARSER).addResultReader(InternalAutoDateHistogram::new)); registerAggregation(new AggregationSpec(GeoDistanceAggregationBuilder.NAME, GeoDistanceAggregationBuilder::new, GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new)); registerAggregation(new AggregationSpec(GeoHashGridAggregationBuilder.NAME, GeoHashGridAggregationBuilder::new, - GeoHashGridAggregationBuilder.PARSER).addResultReader(InternalGeoHashGrid::new)); + GeoHashGridAggregationBuilder.PARSER) + .addResultReader(InternalGeoHashGrid::new) + .setAggregatorRegistrar(GeoHashGridAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(GeoTileGridAggregationBuilder.NAME, GeoTileGridAggregationBuilder::new, - GeoTileGridAggregationBuilder.PARSER).addResultReader(InternalGeoTileGrid::new)); + GeoTileGridAggregationBuilder.PARSER) + .addResultReader(InternalGeoTileGrid::new) + .setAggregatorRegistrar(GeoTileGridAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new, NestedAggregationBuilder::parse).addResultReader(InternalNested::new)); registerAggregation(new AggregationSpec(ReverseNestedAggregationBuilder.NAME, ReverseNestedAggregationBuilder::new, @@ -420,14 +453,21 @@ private void registerAggregations(List plugins) { registerAggregation(new AggregationSpec(TopHitsAggregationBuilder.NAME, TopHitsAggregationBuilder::new, TopHitsAggregationBuilder::parse).addResultReader(InternalTopHits::new)); registerAggregation(new AggregationSpec(GeoBoundsAggregationBuilder.NAME, GeoBoundsAggregationBuilder::new, - GeoBoundsAggregationBuilder.PARSER).addResultReader(InternalGeoBounds::new)); + GeoBoundsAggregationBuilder.PARSER) + .addResultReader(InternalGeoBounds::new) + .setAggregatorRegistrar(GeoBoundsAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(GeoCentroidAggregationBuilder.NAME, GeoCentroidAggregationBuilder::new, - GeoCentroidAggregationBuilder.PARSER).addResultReader(InternalGeoCentroid::new)); + GeoCentroidAggregationBuilder.PARSER) + .addResultReader(InternalGeoCentroid::new) + .setAggregatorRegistrar(GeoCentroidAggregationBuilder::registerAggregators)); registerAggregation(new AggregationSpec(ScriptedMetricAggregationBuilder.NAME, ScriptedMetricAggregationBuilder::new, ScriptedMetricAggregationBuilder.PARSER).addResultReader(InternalScriptedMetric::new)); registerAggregation((new AggregationSpec(CompositeAggregationBuilder.NAME, CompositeAggregationBuilder::new, CompositeAggregationBuilder.PARSER).addResultReader(InternalComposite::new))); registerFromPlugin(plugins, SearchPlugin::getAggregations, this::registerAggregation); + + // after aggs have been registered, see if there are any new VSTypes that need to be linked to core fields + registerFromPlugin(plugins, SearchPlugin::getBareAggregatorRegistrar, this::registerBareAggregatorRegistrar); } private void registerAggregation(AggregationSpec spec) { @@ -442,6 +482,16 @@ private void registerAggregation(AggregationSpec spec) { Writeable.Reader internalReader = t.getValue(); namedWriteables.add(new NamedWriteableRegistry.Entry(InternalAggregation.class, writeableName, internalReader)); } + Consumer register = spec.getAggregatorRegistrar(); + if (register != null) { + register.accept(this.valuesSourceRegistry); + } + } + + private void registerBareAggregatorRegistrar(Consumer registrar) { + if (registrar != null) { + registrar.accept(this.valuesSourceRegistry); + } } private void registerPipelineAggregations(List plugins) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java index 72ac99d94b951..3b658063d17b3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java @@ -105,7 +105,7 @@ private AggregationBuilders() { * Create a new {@link ValueCount} aggregation with the given name. */ public static ValueCountAggregationBuilder count(String name) { - return new ValueCountAggregationBuilder(name, null); + return new ValueCountAggregationBuilder(name); } /** @@ -217,7 +217,7 @@ public static GlobalAggregationBuilder global(String name) { * Create a new {@link Missing} aggregation with the given name. */ public static MissingAggregationBuilder missing(String name) { - return new MissingAggregationBuilder(name, null); + return new MissingAggregationBuilder(name); } /** @@ -266,7 +266,7 @@ public static GeoTileGridAggregationBuilder geotileGrid(String name) { * Create a new {@link SignificantTerms} aggregation with the given name. */ public static SignificantTermsAggregationBuilder significantTerms(String name) { - return new SignificantTermsAggregationBuilder(name, null); + return new SignificantTermsAggregationBuilder(name); } @@ -313,7 +313,7 @@ public static IpRangeAggregationBuilder ipRange(String name) { * Create a new {@link Terms} aggregation with the given name. */ public static TermsAggregationBuilder terms(String name) { - return new TermsAggregationBuilder(name, null); + return new TermsAggregationBuilder(name); } /** @@ -341,7 +341,7 @@ public static MedianAbsoluteDeviationAggregationBuilder medianAbsoluteDeviation( * Create a new {@link Cardinality} aggregation with the given name. */ public static CardinalityAggregationBuilder cardinality(String name) { - return new CardinalityAggregationBuilder(name, null); + return new CardinalityAggregationBuilder(name); } /** diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java index 69d2a145d1349..86acc021cda04 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; @@ -273,11 +274,11 @@ public String format() { * @param config The {@link ValuesSourceConfig} for this source. */ protected abstract CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config) throws IOException; + ValuesSourceConfig config) throws IOException; public final CompositeValuesSourceConfig build(QueryShardContext queryShardContext) throws IOException { - ValuesSourceConfig config = ValuesSourceConfig.resolve(queryShardContext, - valueType, field, script, null, timeZone(), format); + ValuesSourceConfig config = ValuesSourceConfig.resolveUnregistered(queryShardContext, + valueType, field, script, null, timeZone(), format, CoreValuesSourceType.BYTES); return innerBuild(queryShardContext, config); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java index 60092f1d353c5..5ca9fcd2d3b80 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java @@ -26,9 +26,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.AbstractObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.support.ValueType; @@ -37,19 +37,20 @@ import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; public class CompositeValuesSourceParserHelper { + static , T> void declareValuesSourceFields(AbstractObjectParser objectParser, - ValueType targetValueType) { + ValueType expectedValueType) { objectParser.declareField(VB::field, XContentParser::text, new ParseField("field"), ObjectParser.ValueType.STRING); objectParser.declareBoolean(VB::missingBucket, new ParseField("missing_bucket")); objectParser.declareField(VB::valueType, p -> { - ValueType valueType = ValueType.resolveForScript(p.text()); - if (targetValueType != null && valueType.isNotA(targetValueType)) { + ValueType valueType = ValueType.lenientParse(p.text()); + if (expectedValueType != null && valueType.isNotA(expectedValueType)) { throw new ParsingException(p.getTokenLocation(), "Aggregation [" + objectParser.getName() + "] was configured with an incompatible value type [" - + valueType + "]. It can only work on value of type [" - + targetValueType + "]"); + + valueType + "]. It can only work on value of type [" + + expectedValueType + "]"); } return valueType; }, new ParseField("value_type"), ObjectParser.ValueType.STRING); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java index dfac9ee524c0b..805194b690ecb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java @@ -247,9 +247,9 @@ public DateHistogramValuesSourceBuilder offset(long offset) { } @Override - protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { + protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { Rounding rounding = dateHistogramInterval.createRounding(timeZone(), offset); - ValuesSource orig = config.toValuesSource(queryShardContext); + ValuesSource orig = config.toValuesSource(); if (orig == null) { orig = ValuesSource.Numeric.EMPTY; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java index 165a50db60d18..c74d62e623ebe 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java @@ -128,8 +128,8 @@ public boolean equals(Object obj) { } @Override - protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { - ValuesSource orig = config.toValuesSource(queryShardContext); + protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { + ValuesSource orig = config.toValuesSource(); if (orig == null) { orig = ValuesSource.GeoPoint.EMPTY; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java index aa15d5a6947d9..4ef15908ddd7a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java @@ -110,8 +110,8 @@ public HistogramValuesSourceBuilder interval(double interval) { } @Override - protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { - ValuesSource orig = config.toValuesSource(queryShardContext); + protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { + ValuesSource orig = config.toValuesSource(); if (orig == null) { orig = ValuesSource.Numeric.EMPTY; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java index cd88a56614ec5..2954c155e0d7c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java @@ -70,8 +70,8 @@ public String type() { } @Override - protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { - ValuesSource vs = config.toValuesSource(queryShardContext); + protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { + ValuesSource vs = config.toValuesSource(); if (vs == null) { // The field is unmapped so we use a value source that can parse any type of values. // This is needed because the after values are parsed even when there are no values to process. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index 71a79818828ed..88348e2545db6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -40,14 +40,12 @@ import org.elasticsearch.search.aggregations.bucket.BucketUtils; import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; -public abstract class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder +public abstract class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder implements MultiBucketAggregationBuilder { /* recognized field names in JSON */ static final ParseField FIELD_PRECISION = new ParseField("precision"); @@ -68,7 +66,7 @@ protected interface PrecisionParser { public static ObjectParser createParser( String name, PrecisionParser precisionParser, Function ctor) { ObjectParser parser = ObjectParser.fromBuilder(name, ctor); - ValuesSourceParserHelper.declareGeoFields(parser, false, false); + ValuesSourceAggregationBuilder.declareFields(parser, false, false, false); parser.declareField((p, builder, context) -> builder.precision(precisionParser.parse(p)), FIELD_PRECISION, org.elasticsearch.common.xcontent.ObjectParser.ValueType.INT); parser.declareInt(GeoGridAggregationBuilder::size, FIELD_SIZE); @@ -81,7 +79,7 @@ public static ObjectParser crea } public GeoGridAggregationBuilder(String name) { - super(name, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(name); } protected GeoGridAggregationBuilder(GeoGridAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -96,7 +94,7 @@ protected GeoGridAggregationBuilder(GeoGridAggregationBuilder clone, Builder fac * Read from a stream. */ public GeoGridAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(in); precision = in.readVInt(); requiredSize = in.readVInt(); shardSize = in.readVInt(); @@ -105,6 +103,11 @@ public GeoGridAggregationBuilder(StreamInput in) throws IOException { } } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.GEOPOINT; + } + @Override protected void innerWriteTo(StreamOutput out) throws IOException { out.writeVInt(precision); @@ -125,8 +128,8 @@ protected void innerWriteTo(StreamOutput out) throws IOException { /** * Creates a new instance of the {@link ValuesSourceAggregatorFactory}-derived class specific to the geo aggregation. */ - protected abstract ValuesSourceAggregatorFactory createFactory( - String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + protected abstract ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, Builder subFactoriesBuilder, Map metaData ) throws IOException; @@ -172,9 +175,9 @@ public GeoBoundingBox geoBoundingBox() { } @Override - protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, Builder subFactoriesBuilder) + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { int shardSize = this.shardSize; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java index e685a82b00e86..c0193e17ab245 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java @@ -30,9 +30,9 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; public class GeoHashGridAggregationBuilder extends GeoGridAggregationBuilder { public static final String NAME = "geohash_grid"; @@ -53,6 +53,10 @@ public GeoHashGridAggregationBuilder(StreamInput in) throws IOException { super(in); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + GeoHashGridAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + @Override public GeoGridAggregationBuilder precision(int precision) { this.precision = GeoUtils.checkPrecisionRange(precision); @@ -60,8 +64,8 @@ public GeoGridAggregationBuilder precision(int precision) { } @Override - protected ValuesSourceAggregatorFactory createFactory( - String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + protected ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java index 2d7087a693bfa..42d3cc2a39b83 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java @@ -22,15 +22,20 @@ import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.NonCollectingAggregator; +import org.elasticsearch.search.aggregations.metrics.GeoGridAggregatorSupplier; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -38,14 +43,14 @@ import java.util.List; import java.util.Map; -public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory { +public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory { private final int precision; private final int requiredSize; private final int shardSize; private final GeoBoundingBox geoBoundingBox; - GeoHashGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, + GeoHashGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { @@ -72,17 +77,33 @@ public InternalAggregation buildEmptyAggregation() { } @Override - protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, + protected Aggregator doCreateInternal(final ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry() + .getAggregator(config.valueSourceType(), GeoHashGridAggregationBuilder.NAME); + if (aggregatorSupplier instanceof GeoGridAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected " + + GeoGridAggregatorSupplier.class.getName() + ", found [" + aggregatorSupplier.getClass().toString() + "]"); + } if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, geoBoundingBox, Geohash::longEncode); - return new GeoHashGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, - searchContext, parent, pipelineAggregators, metaData); + return ((GeoGridAggregatorSupplier) aggregatorSupplier).build(name, factories, valuesSource, precision, geoBoundingBox, + requiredSize, shardSize, searchContext, parent, pipelineAggregators, metaData); + } + + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(GeoHashGridAggregationBuilder.NAME, CoreValuesSourceType.GEOPOINT, + (GeoGridAggregatorSupplier) (name, factories, valuesSource, precision, geoBoundingBox, requiredSize, shardSize, + aggregationContext, parent, pipelineAggregators, metaData) -> { + CellIdSource cellIdSource = new CellIdSource((ValuesSource.GeoPoint) valuesSource, precision, geoBoundingBox, + Geohash::longEncode); + return new GeoHashGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, aggregationContext, + parent, pipelineAggregators, metaData); + }); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index 01a670caf6cfe..a6e8d3f3135ea 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -29,9 +29,9 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; public class GeoTileGridAggregationBuilder extends GeoGridAggregationBuilder { public static final String NAME = "geotile_grid"; @@ -52,6 +52,10 @@ public GeoTileGridAggregationBuilder(StreamInput in) throws IOException { super(in); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + GeoTileGridAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + @Override public GeoGridAggregationBuilder precision(int precision) { this.precision = GeoTileUtils.checkPrecisionRange(precision); @@ -59,10 +63,10 @@ public GeoGridAggregationBuilder precision(int precision) { } @Override - protected ValuesSourceAggregatorFactory createFactory( - String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + protected ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder, Map metaData ) throws IOException { + AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { return new GeoTileGridAggregatorFactory(name, config, precision, requiredSize, shardSize, geoBoundingBox, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index 0f59c9a71ea40..0039a36ffab92 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -21,15 +21,20 @@ import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.NonCollectingAggregator; +import org.elasticsearch.search.aggregations.metrics.GeoGridAggregatorSupplier; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -37,14 +42,14 @@ import java.util.List; import java.util.Map; -public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory { +public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory { private final int precision; private final int requiredSize; private final int shardSize; private final GeoBoundingBox geoBoundingBox; - GeoTileGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, + GeoTileGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { @@ -71,17 +76,33 @@ public InternalAggregation buildEmptyAggregation() { } @Override - protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, + protected Aggregator doCreateInternal(final ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry() + .getAggregator(config.valueSourceType(), GeoTileGridAggregationBuilder.NAME); + if (aggregatorSupplier instanceof GeoGridAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected " + + GeoGridAggregatorSupplier.class.getName() + ", found [" + aggregatorSupplier.getClass().toString() + "]"); + } if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, geoBoundingBox, GeoTileUtils::longEncode); - return new GeoTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, - searchContext, parent, pipelineAggregators, metaData); + return ((GeoGridAggregatorSupplier) aggregatorSupplier).build(name, factories, valuesSource, precision, geoBoundingBox, + requiredSize, shardSize, searchContext, parent, pipelineAggregators, metaData); + } + + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(GeoTileGridAggregationBuilder.NAME, CoreValuesSourceType.GEOPOINT, + (GeoGridAggregatorSupplier) (name, factories, valuesSource, precision, geoBoundingBox, requiredSize, shardSize, + aggregationContext, parent, pipelineAggregators, metaData) -> { + CellIdSource cellIdSource = new CellIdSource((ValuesSource.GeoPoint) valuesSource, precision, geoBoundingBox, + GeoTileUtils::longEncode); + return new GeoTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, aggregationContext, + parent, pipelineAggregators, metaData); + }); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java index 06cf86d852e11..7ed324ace5955 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java @@ -42,16 +42,13 @@ import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; public class AutoDateHistogramAggregationBuilder - extends ValuesSourceAggregationBuilder { + extends ValuesSourceAggregationBuilder { public static final String NAME = "auto_date_histogram"; @@ -61,7 +58,7 @@ public class AutoDateHistogramAggregationBuilder public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, AutoDateHistogramAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, true); PARSER.declareInt(AutoDateHistogramAggregationBuilder::setNumBuckets, NUM_BUCKETS_FIELD); PARSER.declareStringOrNull(AutoDateHistogramAggregationBuilder::setMinimumIntervalExpression, MINIMUM_INTERVAL_FIELD); } @@ -129,12 +126,12 @@ public AutoDateHistogramAggregationBuilder setMinimumIntervalExpression(String m /** Create a new builder with the given name. */ public AutoDateHistogramAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.DATE); + super(name); } /** Read from a stream, for internal use only. */ public AutoDateHistogramAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.NUMERIC, ValueType.DATE); + super(in); numBuckets = in.readVInt(); if (in.getVersion().onOrAfter(Version.V_7_3_0)) { minimumIntervalExpression = in.readOptionalString(); @@ -148,6 +145,12 @@ protected AutoDateHistogramAggregationBuilder(AutoDateHistogramAggregationBuilde this.minimumIntervalExpression = clone.minimumIntervalExpression; } + @Override + protected ValuesSourceType defaultValueSourceType() { + // TODO: This should probably be DATE, but we're not failing tests with BYTES, so needs more tests? + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new AutoDateHistogramAggregationBuilder(this, factoriesBuilder, metaData); @@ -179,8 +182,8 @@ public int getNumBuckets() { } @Override - protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, - AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, + AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { RoundingInfo[] roundings = buildRoundings(timeZone(), getMinimumIntervalExpression()); int maxRoundingInterval = Arrays.stream(roundings,0, roundings.length-1) .map(rounding -> rounding.innerIntervals) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java index 81af173ab52c9..355b624df8998 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -36,13 +37,13 @@ import java.util.Map; public final class AutoDateHistogramAggregatorFactory - extends ValuesSourceAggregatorFactory { + extends ValuesSourceAggregatorFactory { private final int numBuckets; private RoundingInfo[] roundingInfos; public AutoDateHistogramAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, int numBuckets, RoundingInfo[] roundingInfos, QueryShardContext queryShardContext, @@ -55,16 +56,20 @@ public AutoDateHistogramAggregatorFactory(String name, } @Override - protected Aggregator doCreateInternal(Numeric valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { + if (valuesSource instanceof Numeric == false) { + throw new AggregationExecutionException("ValuesSource type " + valuesSource.toString() + "is not supported for aggregation " + + this.name()); + } if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - return createAggregator(valuesSource, searchContext, parent, pipelineAggregators, metaData); + return createAggregator((Numeric) valuesSource, searchContext, parent, pipelineAggregators, metaData); } private Aggregator createAggregator(ValuesSource.Numeric valuesSource, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java index 03aaaa89f3757..3883aeae58454 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java @@ -47,7 +47,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Relation; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -56,18 +55,16 @@ import org.elasticsearch.search.aggregations.InternalOrder.CompoundOrder; import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.aggregations.support.ValuesSourceType; /** * A builder for histograms on date fields. */ -public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuilder +public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuilder implements MultiBucketAggregationBuilder, DateIntervalConsumer { public static final String NAME = "date_histogram"; @@ -93,7 +90,7 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, DateHistogramAggregationBuilder::new); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, true); DateIntervalWrapper.declareIntervalFields(PARSER); PARSER.declareField(DateHistogramAggregationBuilder::offset, p -> { @@ -115,6 +112,10 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil Histogram.ORDER_FIELD); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + DateHistogramAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + private DateIntervalWrapper dateHistogramInterval = new DateIntervalWrapper(); private long offset = 0; private ExtendedBounds extendedBounds; @@ -124,7 +125,7 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil /** Create a new builder with the given name. */ public DateHistogramAggregationBuilder(String name) { - super(name, CoreValuesSourceType.ANY, ValueType.DATE); + super(name); } protected DateHistogramAggregationBuilder(DateHistogramAggregationBuilder clone, @@ -145,7 +146,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { final ZoneId tz = timeZone(); final Rounding rounding = dateHistogramInterval.createRounding(tz, offset); final ZoneId rewrittenTimeZone = rewriteTimeZone(queryShardContext); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java new file mode 100644 index 0000000000000..fde98b335d3cd --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.histogram; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Rounding; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@FunctionalInterface +public interface DateHistogramAggregationSupplier extends AggregatorSupplier { + Aggregator build(String name, + AggregatorFactories factories, + Rounding rounding, + Rounding shardRounding, + BucketOrder order, + boolean keyed, + long minDocCount, + @Nullable ExtendedBounds extendedBounds, + @Nullable ValuesSource valuesSource, + DocValueFormat formatter, + SearchContext aggregationContext, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java index d68cf814f32bc..8240806bb894a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java @@ -19,25 +19,77 @@ package org.elasticsearch.search.aggregations.bucket.histogram; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Rounding; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -public final class DateHistogramAggregatorFactory - extends ValuesSourceAggregatorFactory { +public final class DateHistogramAggregatorFactory extends ValuesSourceAggregatorFactory { + + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(DateHistogramAggregationBuilder.NAME, + List.of(CoreValuesSourceType.DATE, CoreValuesSourceType.NUMERIC, CoreValuesSourceType.BOOLEAN), + (DateHistogramAggregationSupplier) (String name, + AggregatorFactories factories, + Rounding rounding, + Rounding shardRounding, + BucketOrder order, + boolean keyed, + long minDocCount, + @Nullable ExtendedBounds extendedBounds, + @Nullable ValuesSource valuesSource, + DocValueFormat formatter, + SearchContext aggregationContext, + Aggregator parent, + List pipelineAggregators, + Map metaData) -> new DateHistogramAggregator(name, + factories, rounding, shardRounding, order, keyed, minDocCount, extendedBounds, (ValuesSource.Numeric) valuesSource, + formatter, aggregationContext, parent, pipelineAggregators, metaData)); + + valuesSourceRegistry.register(DateHistogramAggregationBuilder.NAME, + CoreValuesSourceType.RANGE, + (DateHistogramAggregationSupplier) (String name, + AggregatorFactories factories, + Rounding rounding, + Rounding shardRounding, + BucketOrder order, + boolean keyed, + long minDocCount, + @Nullable ExtendedBounds extendedBounds, + @Nullable ValuesSource valuesSource, + DocValueFormat formatter, + SearchContext aggregationContext, + Aggregator parent, + List pipelineAggregators, + Map metaData) -> { + + ValuesSource.Range rangeValueSource = (ValuesSource.Range) valuesSource; + if (rangeValueSource.rangeType() != RangeType.DATE) { + throw new IllegalArgumentException("Expected date range type but found range type [" + rangeValueSource.rangeType().name + + "]"); + } + return new DateRangeHistogramAggregator(name, + factories, rounding, shardRounding, order, keyed, minDocCount, extendedBounds, rangeValueSource, formatter, + aggregationContext, parent, pipelineAggregators, metaData); }); + } private final BucketOrder order; private final boolean keyed; @@ -46,7 +98,7 @@ public final class DateHistogramAggregatorFactory private final Rounding rounding; private final Rounding shardRounding; - public DateHistogramAggregatorFactory(String name, ValuesSourceConfig config, + public DateHistogramAggregatorFactory(String name, ValuesSourceConfig config, BucketOrder order, boolean keyed, long minDocCount, Rounding rounding, Rounding shardRounding, ExtendedBounds extendedBounds, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, @@ -64,15 +116,6 @@ public long minDocCount() { return minDocCount; } - @Override - protected ValuesSource resolveMissingAny(Object missing) { - if (missing instanceof Number) { - return ValuesSource.Numeric.EMPTY; - } - throw new IllegalArgumentException("Only numeric missing values are supported for date histogram aggregation, found [" - + missing + "]"); - } - @Override protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, @@ -83,36 +126,14 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - if (valuesSource instanceof ValuesSource.Numeric) { - return createAggregator((ValuesSource.Numeric) valuesSource, searchContext, parent, pipelineAggregators, metaData); - } else if (valuesSource instanceof ValuesSource.Range) { - ValuesSource.Range rangeValueSource = (ValuesSource.Range) valuesSource; - if (rangeValueSource.rangeType() != RangeType.DATE) { - throw new IllegalArgumentException("Expected date range type but found range type [" + rangeValueSource.rangeType().name - + "]"); - } - return createRangeAggregator((ValuesSource.Range) valuesSource, searchContext, parent, pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + DateHistogramAggregationBuilder.NAME); + if (aggregatorSupplier instanceof DateHistogramAggregationSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected DateHistogramAggregationSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); } - else { - throw new IllegalArgumentException("Expected one of [Date, Range] values source, found [" - + valuesSource.toString() + "]"); - } - } - - private Aggregator createAggregator(ValuesSource.Numeric valuesSource, SearchContext searchContext, - Aggregator parent, List pipelineAggregators, - Map metaData) throws IOException { - return new DateHistogramAggregator(name, factories, rounding, shardRounding, order, keyed, minDocCount, extendedBounds, - valuesSource, config.format(), searchContext, parent, pipelineAggregators, metaData); - } - - private Aggregator createRangeAggregator(ValuesSource.Range valuesSource, - SearchContext searchContext, - Aggregator parent, - List pipelineAggregators, - Map metaData) throws IOException { - return new DateRangeHistogramAggregator(name, factories, rounding, shardRounding, order, keyed, minDocCount, extendedBounds, - valuesSource, config.format(), searchContext, parent, pipelineAggregators, metaData); + return ((DateHistogramAggregationSupplier) aggregatorSupplier).build(name, factories, rounding, shardRounding, order, keyed, + minDocCount, extendedBounds, valuesSource, config.format(), searchContext, parent, pipelineAggregators, metaData); } @Override @@ -120,6 +141,7 @@ protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, List pipelineAggregators, Map metaData) throws IOException { - return createAggregator(null, searchContext, parent, pipelineAggregators, metaData); + return new DateHistogramAggregator(name, factories, rounding, shardRounding, order, keyed, minDocCount, extendedBounds, + null, config.format(), searchContext, parent, pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java index 1a5a2cce36c86..409ca559da116 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -39,19 +38,17 @@ import org.elasticsearch.search.aggregations.InternalOrder.CompoundOrder; import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.aggregations.support.ValuesSourceType; /** * A builder for histograms on numeric fields. This builder can operate on either base numeric fields, or numeric range fields. IP range * fields are unsupported, and will throw at the factory layer. */ -public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder +public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder implements MultiBucketAggregationBuilder { public static final String NAME = "histogram"; @@ -66,7 +63,7 @@ public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder< public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, HistogramAggregationBuilder::new); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); PARSER.declareDouble(HistogramAggregationBuilder::interval, Histogram.INTERVAL_FIELD); @@ -84,6 +81,10 @@ public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder< Histogram.ORDER_FIELD); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + HistogramAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + private double interval; private double offset = 0; private double minBound = Double.POSITIVE_INFINITY; @@ -93,14 +94,13 @@ public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder< private long minDocCount = 0; @Override - protected ValuesSourceType resolveScriptAny(Script script) { - // TODO: No idea how we'd support Range scripts here. + protected ValuesSourceType defaultValueSourceType() { return CoreValuesSourceType.NUMERIC; } /** Create a new builder with the given name. */ public HistogramAggregationBuilder(String name) { - super(name, CoreValuesSourceType.ANY, ValueType.NUMERIC); + super(name); } protected HistogramAggregationBuilder(HistogramAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -121,7 +121,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return new HistogramAggregatorFactory(name, config, interval, offset, order, keyed, minDocCount, minBound, maxBound, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java index 670b8008bc512..347834f5fb825 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java @@ -20,14 +20,20 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.HistogramAggregatorSupplier; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -38,7 +44,7 @@ * Constructs the per-shard aggregator instance for histogram aggregation. Selects the numeric or range field implementation based on the * field type. */ -public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFactory { +public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFactory { private final double interval, offset; private final BucketOrder order; @@ -46,17 +52,45 @@ public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFact private final long minDocCount; private final double minBound, maxBound; - @Override - protected ValuesSource resolveMissingAny(Object missing) { - if (missing instanceof Number) { - return ValuesSource.Numeric.EMPTY; - } - throw new IllegalArgumentException("Only numeric missing values are supported for histogram aggregation, found [" - + missing + "]"); + // TODO: Registration should happen on the actual aggregator classes, but I don't want to set up the whole dynamic loading thing yet + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(HistogramAggregationBuilder.NAME, CoreValuesSourceType.RANGE, + new HistogramAggregatorSupplier() { + @Override + public Aggregator build(String name, AggregatorFactories factories, double interval, double offset, + BucketOrder order, boolean keyed, long minDocCount, double minBound, double maxBound, + ValuesSource valuesSource, DocValueFormat formatter, SearchContext context, + Aggregator parent, List pipelineAggregators, + Map metaData) throws IOException { + ValuesSource.Range rangeValueSource = (ValuesSource.Range) valuesSource; + if (rangeValueSource.rangeType().isNumeric() == false) { + throw new IllegalArgumentException("Expected numeric range type but found non-numeric range [" + + rangeValueSource.rangeType().name + "]"); + } + return new RangeHistogramAggregator(name, factories, interval, offset, order, keyed, minDocCount, minBound, + maxBound, rangeValueSource, formatter, context, parent, pipelineAggregators, metaData); + } + } + ); + + valuesSourceRegistry.register(HistogramAggregationBuilder.NAME, + List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + new HistogramAggregatorSupplier() { + @Override + public Aggregator build(String name, AggregatorFactories factories, double interval, double offset, + BucketOrder order, boolean keyed, long minDocCount, double minBound, double maxBound, + ValuesSource valuesSource, DocValueFormat formatter, SearchContext context, + Aggregator parent, List pipelineAggregators, + Map metaData) throws IOException { + return new NumericHistogramAggregator(name, factories, interval, offset, order, keyed, minDocCount, minBound, + maxBound, (ValuesSource.Numeric) valuesSource, formatter, context, parent, pipelineAggregators, metaData); + } + } + ); } public HistogramAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, double interval, double offset, BucketOrder order, @@ -92,23 +126,16 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - if (valuesSource instanceof ValuesSource.Numeric) { - return new NumericHistogramAggregator(name, factories, interval, offset, order, keyed, minDocCount, minBound, maxBound, - (ValuesSource.Numeric) valuesSource, config.format(), searchContext, parent, pipelineAggregators, metaData); - } else if (valuesSource instanceof ValuesSource.Range) { - ValuesSource.Range rangeValueSource = (ValuesSource.Range) valuesSource; - if (rangeValueSource.rangeType().isNumeric() == false) { - throw new IllegalArgumentException("Expected numeric range type but found non-numeric range [" - + rangeValueSource.rangeType().name + "]"); - } - return new RangeHistogramAggregator(name, factories, interval, offset, order, keyed, minDocCount, minBound, maxBound, - (ValuesSource.Range) valuesSource, config.format(), searchContext, parent, pipelineAggregators, - metaData); - } - else { - throw new IllegalArgumentException("Expected one of [Numeric, Range] values source, found [" - + valuesSource.toString() + "]"); + + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + HistogramAggregationBuilder.NAME); + if (aggregatorSupplier instanceof HistogramAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected HistogramAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); } + HistogramAggregatorSupplier histogramAggregatorSupplier = (HistogramAggregatorSupplier) aggregatorSupplier; + return histogramAggregatorSupplier.build(name, factories, interval, offset, order, keyed, minDocCount, minBound, maxBound, + valuesSource, config.format(), searchContext, parent, pipelineAggregators, metaData); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java index 4ccd0a4dfe616..83e8b1beed77d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java @@ -51,7 +51,7 @@ * written as {@code interval * x + offset} and yet is less than or equal to * {@code value}. */ -class NumericHistogramAggregator extends BucketsAggregator { +public class NumericHistogramAggregator extends BucketsAggregator { private final ValuesSource.Numeric valuesSource; private final DocValueFormat formatter; @@ -63,7 +63,7 @@ class NumericHistogramAggregator extends BucketsAggregator { private final LongHash bucketOrds; - NumericHistogramAggregator(String name, AggregatorFactories factories, double interval, double offset, + public NumericHistogramAggregator(String name, AggregatorFactories factories, double interval, double offset, BucketOrder order, boolean keyed, long minDocCount, double minBound, double maxBound, @Nullable ValuesSource.Numeric valuesSource, DocValueFormat formatter, SearchContext context, Aggregator parent, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java index 57949424aae26..6da1c3bb6192c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java @@ -57,7 +57,7 @@ public class RangeHistogramAggregator extends BucketsAggregator { private final LongHash bucketOrds; - RangeHistogramAggregator(String name, AggregatorFactories factories, double interval, double offset, + public RangeHistogramAggregator(String name, AggregatorFactories factories, double interval, double offset, BucketOrder order, boolean keyed, long minDocCount, double minBound, double maxBound, @Nullable ValuesSource.Range valuesSource, DocValueFormat formatter, SearchContext context, Aggregator parent, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java index 1e0ebf4960d72..d0a631240367c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java @@ -29,33 +29,36 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; -public class MissingAggregationBuilder extends ValuesSourceAggregationBuilder { +public class MissingAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "missing"; public static final ObjectParser PARSER = - ObjectParser.fromBuilder(NAME, name -> new MissingAggregationBuilder(name, null)); + ObjectParser.fromBuilder(NAME, MissingAggregationBuilder::new); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); } - public MissingAggregationBuilder(String name, ValueType targetValueType) { - super(name, CoreValuesSourceType.ANY, targetValueType); + public MissingAggregationBuilder(String name) { + super(name); } protected MissingAggregationBuilder(MissingAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new MissingAggregationBuilder(this, factoriesBuilder, metaData); @@ -65,7 +68,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return new MissingAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } @@ -90,7 +93,7 @@ protected ValuesSourceAggregatorFactory innerBuild(QueryShardConte public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { return builder; } - + @Override public String getType() { return NAME; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java index 6eee7739c4560..b36873875bb1a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java @@ -33,10 +33,11 @@ import java.util.List; import java.util.Map; -public class MissingAggregatorFactory extends ValuesSourceAggregatorFactory { +public class MissingAggregatorFactory extends ValuesSourceAggregatorFactory { - public MissingAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, - AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { + public MissingAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, + AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java index d60851a2d7fef..6e7974f68d605 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java @@ -20,41 +20,70 @@ package org.elasticsearch.search.aggregations.bucket.range; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Unmapped; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -public class AbstractRangeAggregatorFactory extends ValuesSourceAggregatorFactory { +public class AbstractRangeAggregatorFactory extends ValuesSourceAggregatorFactory { private final InternalRange.Factory rangeFactory; private final R[] ranges; private final boolean keyed; + private final String aggregationTypeName; + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry, String aggregationName) { + valuesSourceRegistry.register(aggregationName, + List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + new RangeAggregatorSupplier() { + @Override + public Aggregator build(String name, + AggregatorFactories factories, + Numeric valuesSource, + DocValueFormat format, + InternalRange.Factory rangeFactory, + Range[] ranges, + boolean keyed, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException { + return new RangeAggregator(name, factories, valuesSource, format, rangeFactory, ranges, keyed, context, parent, + pipelineAggregators, metaData); + } + }); + } public AbstractRangeAggregatorFactory(String name, - ValuesSourceConfig config, - R[] ranges, - boolean keyed, - InternalRange.Factory rangeFactory, - QueryShardContext queryShardContext, - AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder, - Map metaData) throws IOException { + String aggregationTypeName, + ValuesSourceConfig config, + R[] ranges, + boolean keyed, + InternalRange.Factory rangeFactory, + QueryShardContext queryShardContext, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); this.ranges = ranges; this.keyed = keyed; this.rangeFactory = rangeFactory; + this.aggregationTypeName = aggregationTypeName; } @Override @@ -66,15 +95,20 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(Numeric valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new RangeAggregator(name, factories, valuesSource, config.format(), rangeFactory, ranges, keyed, searchContext, parent, - pipelineAggregators, metaData); - } - + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + aggregationTypeName); + if (aggregatorSupplier instanceof RangeAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected RangeAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((RangeAggregatorSupplier)aggregatorSupplier).build(name, factories, (Numeric) valuesSource, config.format(), rangeFactory, + ranges, keyed, searchContext, parent, pipelineAggregators, metaData); + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java index 542929564e66f..4e8e2acaf88fe 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java @@ -27,8 +27,8 @@ import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -38,14 +38,14 @@ import java.util.function.Function; public abstract class AbstractRangeBuilder, R extends Range> - extends ValuesSourceAggregationBuilder implements MultiBucketAggregationBuilder { + extends ValuesSourceAggregationBuilder implements MultiBucketAggregationBuilder { protected final InternalRange.Factory rangeFactory; protected List ranges = new ArrayList<>(); protected boolean keyed = false; protected AbstractRangeBuilder(String name, InternalRange.Factory rangeFactory) { - super(name, rangeFactory.getValueSourceType(), rangeFactory.getValueType()); + super(name); this.rangeFactory = rangeFactory; } @@ -62,12 +62,18 @@ protected AbstractRangeBuilder(AbstractRangeBuilder clone, */ protected AbstractRangeBuilder(StreamInput in, InternalRange.Factory rangeFactory, Writeable.Reader rangeReader) throws IOException { - super(in, rangeFactory.getValueSourceType(), rangeFactory.getValueType()); + super(in); this.rangeFactory = rangeFactory; ranges = in.readList(rangeReader); keyed = in.readBoolean(); } + @Override + protected ValuesSourceType defaultValueSourceType() { + // Copied over from the old targetValueType setting. Not sure what cases this is still relevant for. --Tozzi 2020-01-13 + return rangeFactory.getValueSourceType(); + } + /** * Resolve any strings in the ranges so we have a number value for the from * and to of each range. The ranges are also sorted before being returned. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java index e11c8d201b4b0..60237ed3a36cb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.range; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -33,13 +34,13 @@ import java.util.Map; public class BinaryRangeAggregatorFactory - extends ValuesSourceAggregatorFactory { + extends ValuesSourceAggregatorFactory { private final List ranges; private final boolean keyed; public BinaryRangeAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, List ranges, boolean keyed, QueryShardContext queryShardContext, AggregatorFactory parent, Builder subFactoriesBuilder, @@ -58,12 +59,16 @@ protected Aggregator createUnmapped(SearchContext searchContext, Aggregator pare } @Override - protected Aggregator doCreateInternal(ValuesSource.Bytes valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new BinaryRangeAggregator(name, factories, valuesSource, config.format(), + if (valuesSource instanceof ValuesSource.Bytes == false) { + throw new AggregationExecutionException("ValuesSource type " + valuesSource.toString() + "is not supported for aggregation " + + this.name()); + } + return new BinaryRangeAggregator(name, factories, (ValuesSource.Bytes) valuesSource, config.format(), ranges, keyed, searchContext, parent, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java index 92880c2011df0..ef5c449ef742c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java @@ -27,9 +27,11 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZonedDateTime; @@ -41,7 +43,7 @@ public class DateRangeAggregationBuilder extends AbstractRangeBuilder PARSER = ObjectParser.fromBuilder(NAME, DateRangeAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, true); PARSER.declareBoolean(DateRangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD); PARSER.declareObjectArray((agg, ranges) -> { @@ -56,6 +58,10 @@ private static RangeAggregator.Range parseRange(XContentParser parser) throws IO return RangeAggregator.Range.fromXContent(parser); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + AbstractRangeAggregatorFactory.registerAggregators(valuesSourceRegistry, NAME); + } + public DateRangeAggregationBuilder(String name) { super(name, InternalDateRange.FACTORY); } @@ -81,6 +87,11 @@ public String getType() { return NAME; } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.DATE; + } + /** * Add a new range to this aggregation. * @@ -285,7 +296,7 @@ public DateRangeAggregationBuilder addUnboundedFrom(ZonedDateTime from) { } @Override - protected DateRangeAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, + protected DateRangeAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { // We need to call processRanges here so they are parsed and we know whether `now` has been used before we make // the decision of whether to cache the request diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorFactory.java index 8c06709c2f04f..c981c13c839a9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorFactory.java @@ -22,7 +22,6 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; @@ -31,15 +30,16 @@ public class DateRangeAggregatorFactory extends AbstractRangeAggregatorFactory { public DateRangeAggregatorFactory(String name, - ValuesSourceConfig config, - RangeAggregator.Range[] ranges, - boolean keyed, - InternalRange.Factory rangeFactory, - QueryShardContext queryShardContext, - AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder, - Map metaData) throws IOException { - super(name, config, ranges, keyed, rangeFactory, queryShardContext, parent, subFactoriesBuilder, metaData); + ValuesSourceConfig config, + RangeAggregator.Range[] ranges, + boolean keyed, + InternalRange.Factory rangeFactory, + QueryShardContext queryShardContext, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { + super(name, DateRangeAggregationBuilder.NAME, config, ranges, keyed, rangeFactory, queryShardContext, parent, subFactoriesBuilder, + metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java index 92d8bbc15a754..0d85cebce3069 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java @@ -35,11 +35,11 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -51,7 +51,7 @@ import static org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range.KEY_FIELD; import static org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range.TO_FIELD; -public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilder { +public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "geo_distance"; static final ParseField ORIGIN_FIELD = new ParseField("origin", "center", "point", "por"); static final ParseField UNIT_FIELD = new ParseField("unit"); @@ -60,7 +60,7 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde private static final ObjectParser PARSER; static { PARSER = new ObjectParser<>(GeoDistanceAggregationBuilder.NAME); - ValuesSourceParserHelper.declareGeoFields(PARSER, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, false, false); PARSER.declareBoolean(GeoDistanceAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD); @@ -230,15 +230,16 @@ public GeoDistanceAggregationBuilder(String name, GeoPoint origin) { private GeoDistanceAggregationBuilder(String name, GeoPoint origin, InternalRange.Factory rangeFactory) { - super(name, rangeFactory.getValueSourceType(), rangeFactory.getValueType()); + super(name); this.origin = origin; } + /** * Read from a stream. */ public GeoDistanceAggregationBuilder(StreamInput in) throws IOException { - super(in, InternalGeoDistance.FACTORY.getValueSourceType(), InternalGeoDistance.FACTORY.getValueType()); + super(in); origin = new GeoPoint(in.readDouble(), in.readDouble()); int size = in.readVInt(); ranges = new ArrayList<>(size); @@ -264,6 +265,13 @@ protected GeoDistanceAggregationBuilder(GeoDistanceAggregationBuilder clone, Bui this.ranges = new ArrayList<>(clone.ranges); } + @Override + protected ValuesSourceType defaultValueSourceType() { + // TODO: This should probably not be BYTES, but we're not failing tests with BYTES, so needs more tests? + // TODO: this should set defaultValuesSourceType to GEOPOINT + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new GeoDistanceAggregationBuilder(this, factoriesBuilder, metaData); @@ -411,10 +419,10 @@ public boolean keyed() { } @Override - protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { Range[] ranges = this.ranges.toArray(new Range[this.range().size()]); if (ranges.length == 0) { throw new IllegalArgumentException("No [ranges] specified for the [" + this.getName() + "] aggregation"); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java index 711297762b8d4..a073c5c799d70 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -44,7 +45,7 @@ import java.util.Map; public class GeoDistanceRangeAggregatorFactory - extends ValuesSourceAggregatorFactory { + extends ValuesSourceAggregatorFactory { private final InternalRange.Factory rangeFactory = InternalGeoDistance.FACTORY; private final GeoPoint origin; @@ -53,9 +54,11 @@ public class GeoDistanceRangeAggregatorFactory private final GeoDistance distanceType; private final boolean keyed; - public GeoDistanceRangeAggregatorFactory(String name, ValuesSourceConfig config, GeoPoint origin, - Range[] ranges, DistanceUnit unit, GeoDistance distanceType, boolean keyed, QueryShardContext queryShardContext, - AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { + public GeoDistanceRangeAggregatorFactory(String name, ValuesSourceConfig config, GeoPoint origin, + Range[] ranges, DistanceUnit unit, GeoDistance distanceType, boolean keyed, + QueryShardContext queryShardContext, AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); this.origin = origin; this.ranges = ranges; @@ -74,13 +77,17 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, + protected Aggregator doCreateInternal(final ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - DistanceSource distanceSource = new DistanceSource(valuesSource, distanceType, origin, unit); + if (valuesSource instanceof ValuesSource.GeoPoint == false) { + throw new AggregationExecutionException("ValuesSource type " + valuesSource.toString() + "is not supported for aggregation " + + this.name()); + } + DistanceSource distanceSource = new DistanceSource((ValuesSource.GeoPoint) valuesSource, distanceType, origin, unit); return new RangeAggregator(name, factories, distanceSource, config.format(), rangeFactory, ranges, keyed, searchContext, parent, pipelineAggregators, metaData); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java index eda8c0524e6bf..b08a50a8289d9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java @@ -46,23 +46,21 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; public final class IpRangeAggregationBuilder - extends ValuesSourceAggregationBuilder { + extends ValuesSourceAggregationBuilder { public static final String NAME = "ip_range"; private static final ParseField MASK_FIELD = new ParseField("mask"); public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, IpRangeAggregationBuilder::new); static { - ValuesSourceParserHelper.declareBytesFields(PARSER, false, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, false, false, false); PARSER.declareBoolean(IpRangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD); @@ -216,7 +214,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws private List ranges = new ArrayList<>(); public IpRangeAggregationBuilder(String name) { - super(name, CoreValuesSourceType.BYTES, ValueType.IP); + super(name); } protected IpRangeAggregationBuilder(IpRangeAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -333,7 +331,7 @@ public IpRangeAggregationBuilder addUnboundedFrom(String from) { } public IpRangeAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.BYTES, ValueType.IP); + super(in); final int numRanges = in.readVInt(); for (int i = 0; i < numRanges; ++i) { addRange(new Range(in)); @@ -341,6 +339,11 @@ public IpRangeAggregationBuilder(StreamInput in) throws IOException { keyed = in.readBoolean(); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.IP; + } + @Override protected void innerWriteTo(StreamOutput out) throws IOException { out.writeVInt(ranges.size()); @@ -360,8 +363,8 @@ private static BytesRef toBytesRef(String ip) { } @Override - protected ValuesSourceAggregatorFactory innerBuild( - QueryShardContext queryShardContext, ValuesSourceConfig config, + protected ValuesSourceAggregatorFactory innerBuild( + QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { List ranges = new ArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java index 7744cae764c57..465ff434cd5df 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java @@ -28,9 +28,9 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import java.io.IOException; import java.util.Map; @@ -41,7 +41,7 @@ public class RangeAggregationBuilder extends AbstractRangeBuilder PARSER = ObjectParser.fromBuilder(NAME, RangeAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); PARSER.declareBoolean(RangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD); PARSER.declareObjectArray((agg, ranges) -> { @@ -55,6 +55,10 @@ private static Range parseRange(XContentParser parser) throws IOException { return Range.fromXContent(parser); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + AbstractRangeAggregatorFactory.registerAggregators(valuesSourceRegistry, NAME); + } + public RangeAggregationBuilder(String name) { super(name, InternalRange.FACTORY); } @@ -142,7 +146,7 @@ public RangeAggregationBuilder addUnboundedFrom(double from) { } @Override - protected RangeAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, + protected RangeAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { // We need to call processRanges here so they are parsed before we make the decision of whether to cache the request Range[] ranges = processRanges(range -> { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java index 4fe886e0e6147..7d30605d19646 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java @@ -24,7 +24,6 @@ import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.bucket.range.InternalRange.Factory; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; @@ -32,10 +31,17 @@ public class RangeAggregatorFactory extends AbstractRangeAggregatorFactory { - public RangeAggregatorFactory(String name, ValuesSourceConfig config, Range[] ranges, boolean keyed, - Factory rangeFactory, QueryShardContext queryShardContext, AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { - super(name, config, ranges, keyed, rangeFactory, queryShardContext, parent, subFactoriesBuilder, metaData); + public RangeAggregatorFactory(String name, + ValuesSourceConfig config, + Range[] ranges, + boolean keyed, + Factory rangeFactory, + QueryShardContext queryShardContext, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { + super(name, RangeAggregationBuilder.NAME, config, ranges, keyed, rangeFactory, queryShardContext, parent, subFactoriesBuilder, + metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java new file mode 100644 index 0000000000000..2594396e33a3f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.range; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface RangeAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + AggregatorFactories factories, + ValuesSource.Numeric valuesSource, + DocValueFormat format, + InternalRange.Factory rangeFactory, + RangeAggregator.Range[] ranges, + boolean keyed, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java index 62bfa453a54ef..124e4e017dcd7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java @@ -28,17 +28,16 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; import java.util.Objects; -public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilder { +public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "diversified_sampler"; public static final int MAX_DOCS_PER_VALUE_DEFAULT = 1; @@ -46,7 +45,7 @@ public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilde public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, DiversifiedAggregationBuilder::new); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, false, false); PARSER.declareInt(DiversifiedAggregationBuilder::shardSize, SamplerAggregator.SHARD_SIZE_FIELD); PARSER.declareInt(DiversifiedAggregationBuilder::maxDocsPerValue, SamplerAggregator.MAX_DOCS_PER_VALUE_FIELD); PARSER.declareString(DiversifiedAggregationBuilder::executionHint, SamplerAggregator.EXECUTION_HINT_FIELD); @@ -57,7 +56,7 @@ public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilde private String executionHint = null; public DiversifiedAggregationBuilder(String name) { - super(name, CoreValuesSourceType.ANY, null); + super(name); } protected DiversifiedAggregationBuilder(DiversifiedAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -67,6 +66,11 @@ protected DiversifiedAggregationBuilder(DiversifiedAggregationBuilder clone, Bui this.executionHint = clone.executionHint; } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new DiversifiedAggregationBuilder(this, factoriesBuilder, metaData); @@ -76,7 +80,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return new DiversifiedAggregatorFactory(name, config, shardSize, maxDocsPerValue, executionHint, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java index 84ee57ccb08a6..60a66b10cd553 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java @@ -38,13 +38,13 @@ import java.util.List; import java.util.Map; -public class DiversifiedAggregatorFactory extends ValuesSourceAggregatorFactory { +public class DiversifiedAggregatorFactory extends ValuesSourceAggregatorFactory { private final int shardSize; private final int maxDocsPerValue; private final String executionHint; - DiversifiedAggregatorFactory(String name, ValuesSourceConfig config, int shardSize, int maxDocsPerValue, + DiversifiedAggregatorFactory(String name, ValuesSourceConfig config, int shardSize, int maxDocsPerValue, String executionHint, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java index 45a755d07b286..702f929a2fe2b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java @@ -39,12 +39,11 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -52,7 +51,7 @@ import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; -public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationBuilder +public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationBuilder implements MultiBucketAggregationBuilder { public static final String NAME = "significant_terms"; @@ -67,7 +66,7 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB SignificantTermsAggregationBuilder.NAME, SignificanceHeuristic.class, SignificantTermsAggregationBuilder::significanceHeuristic, null); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); PARSER.declareInt(SignificantTermsAggregationBuilder::shardSize, TermsAggregationBuilder.SHARD_SIZE_FIELD_NAME); @@ -92,7 +91,11 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB IncludeExclude::parseExclude, IncludeExclude.EXCLUDE_FIELD, ObjectParser.ValueType.STRING_ARRAY); } public static SignificantTermsAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException { - return PARSER.parse(parser, new SignificantTermsAggregationBuilder(aggregationName, null), null); + return PARSER.parse(parser, new SignificantTermsAggregationBuilder(aggregationName), null); + } + + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + SignificantTermsAggregatorFactory.registerAggregators(valuesSourceRegistry); } private IncludeExclude includeExclude = null; @@ -101,15 +104,15 @@ public static SignificantTermsAggregationBuilder parse(String aggregationName, X private TermsAggregator.BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(DEFAULT_BUCKET_COUNT_THRESHOLDS); private SignificanceHeuristic significanceHeuristic = DEFAULT_SIGNIFICANCE_HEURISTIC; - public SignificantTermsAggregationBuilder(String name, ValueType valueType) { - super(name, CoreValuesSourceType.ANY, valueType); + public SignificantTermsAggregationBuilder(String name) { + super(name); } /** * Read from a Stream. */ public SignificantTermsAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.ANY); + super(in); bucketCountThresholds = new BucketCountThresholds(in); executionHint = in.readOptionalString(); filterBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); @@ -127,12 +130,16 @@ protected SignificantTermsAggregationBuilder(SignificantTermsAggregationBuilder this.significanceHeuristic = clone.significanceHeuristic; } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected SignificantTermsAggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new SignificantTermsAggregationBuilder(this, factoriesBuilder, metaData); } - @Override protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException { if (filterBuilder != null) { QueryBuilder rewrittenFilter = filterBuilder.rewrite(queryShardContext); @@ -283,10 +290,10 @@ public SignificanceHeuristic significanceHeuristic() { } @Override - protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { SignificanceHeuristic executionHeuristic = this.significanceHeuristic.rewrite(queryShardContext); return new SignificantTermsAggregatorFactory(name, config, includeExclude, executionHint, filterBuilder, bucketCountThresholds, executionHeuristic, queryShardContext, parent, subFactoriesBuilder, metaData); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java index 3de703099bcb2..8515fa96fd336 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java @@ -50,17 +50,20 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFactory - implements Releasable { +public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFactory + implements Releasable { private static final DeprecationLogger deprecationLogger = new DeprecationLogger( LogManager.getLogger(SignificantTermsAggregatorFactory.class)); @@ -75,17 +78,116 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac private final TermsAggregator.BucketCountThresholds bucketCountThresholds; private final SignificanceHeuristic significanceHeuristic; - public SignificantTermsAggregatorFactory(String name, - ValuesSourceConfig config, - IncludeExclude includeExclude, - String executionHint, - QueryBuilder filterBuilder, - TermsAggregator.BucketCountThresholds bucketCountThresholds, - SignificanceHeuristic significanceHeuristic, - QueryShardContext queryShardContext, - AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder, - Map metaData) throws IOException { + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(SignificantTermsAggregationBuilder.NAME, + List.of(CoreValuesSourceType.BYTES, CoreValuesSourceType.IP), + SignificantTermsAggregatorFactory.bytesSupplier()); + + valuesSourceRegistry.register(SignificantTermsAggregationBuilder.NAME, + List.of(CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN, CoreValuesSourceType.NUMERIC), + SignificantTermsAggregatorFactory.numericSupplier()); + } + + /** + * This supplier is used for all the field types that should be aggregated as bytes/strings, + * including those that need global ordinals + */ + private static SignificantTermsAggregatorSupplier bytesSupplier() { + return new SignificantTermsAggregatorSupplier() { + @Override + public Aggregator build(String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + TermsAggregator.BucketCountThresholds bucketCountThresholds, + IncludeExclude includeExclude, + String executionHint, + SearchContext context, + Aggregator parent, + SignificanceHeuristic significanceHeuristic, + SignificantTermsAggregatorFactory sigTermsFactory, + List pipelineAggregators, + Map metaData) throws IOException { + + ExecutionMode execution = null; + if (executionHint != null) { + execution = ExecutionMode.fromString(executionHint, deprecationLogger); + } + if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals == false) { + execution = ExecutionMode.MAP; + } + if (execution == null) { + execution = ExecutionMode.GLOBAL_ORDINALS; + } + + if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) { + throw new IllegalArgumentException("Aggregation [" + name + "] cannot support regular expression style " + + "include/exclude settings as they can only be applied to string fields. Use an array of values for " + + "include/exclude clauses"); + } + + return execution.create(name, factories, valuesSource, format, bucketCountThresholds, includeExclude, context, parent, + significanceHeuristic, sigTermsFactory, pipelineAggregators, metaData); + + } + }; + } + + /** + * This supplier is used for all fields that expect to be aggregated as a numeric value. + * This includes floating points, and formatted types that use numerics internally for storage (date, boolean, etc) + */ + private static SignificantTermsAggregatorSupplier numericSupplier() { + return new SignificantTermsAggregatorSupplier() { + @Override + public Aggregator build(String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + TermsAggregator.BucketCountThresholds bucketCountThresholds, + IncludeExclude includeExclude, + String executionHint, + SearchContext context, + Aggregator parent, + SignificanceHeuristic significanceHeuristic, + SignificantTermsAggregatorFactory sigTermsFactory, + List pipelineAggregators, + Map metaData) throws IOException { + + if ((includeExclude != null) && (includeExclude.isRegexBased())) { + throw new IllegalArgumentException("Aggregation [" + name + "] cannot support regular expression style include/exclude " + + "settings as they can only be applied to string fields. Use an array of numeric " + + "values for include/exclude clauses used to filter numeric fields"); + } + + if (((ValuesSource.Numeric) valuesSource).isFloatingPoint()) { + throw new UnsupportedOperationException("No support for examining floating point numerics"); + } + + IncludeExclude.LongFilter longFilter = null; + if (includeExclude != null) { + longFilter = includeExclude.convertToLongFilter(format); + } + + return new SignificantLongTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, format, + bucketCountThresholds, context, parent, significanceHeuristic, sigTermsFactory, longFilter, pipelineAggregators, + metaData); + + } + }; + } + + SignificantTermsAggregatorFactory(String name, + ValuesSourceConfig config, + IncludeExclude includeExclude, + String executionHint, + QueryBuilder filterBuilder, + TermsAggregator.BucketCountThresholds bucketCountThresholds, + SignificanceHeuristic significanceHeuristic, + QueryShardContext queryShardContext, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); if (config.unmapped() == false) { @@ -116,7 +218,7 @@ public SignificantTermsAggregatorFactory(String name, /** * Get the number of docs in the superset. */ - public long getSupersetNumDocs() { + long getSupersetNumDocs() { return supersetNumDocs; } @@ -160,12 +262,12 @@ private long getBackgroundFrequency(String value) throws IOException { return queryShardContext.searcher().count(query); } - public long getBackgroundFrequency(BytesRef termBytes) throws IOException { + long getBackgroundFrequency(BytesRef termBytes) throws IOException { String value = config.format().format(termBytes).toString(); return getBackgroundFrequency(value); } - public long getBackgroundFrequency(long termNum) throws IOException { + long getBackgroundFrequency(long termNum) throws IOException { String value = config.format().format(termNum).toString(); return getBackgroundFrequency(value); } @@ -196,6 +298,13 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, return asMultiBucketAggregator(this, searchContext, parent); } + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + SignificantTermsAggregationBuilder.NAME); + if (aggregatorSupplier instanceof SignificantTermsAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected SignificantTermsAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + numberOfAggregatorsCreated++; BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds); if (bucketCountThresholds.getShardSize() == SignificantTermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) { @@ -214,52 +323,12 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, bucketCountThresholds.setShardSize(2 * BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize())); } - if (valuesSource instanceof ValuesSource.Bytes) { - ExecutionMode execution = null; - if (executionHint != null) { - execution = ExecutionMode.fromString(executionHint, deprecationLogger); - } - if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals == false) { - execution = ExecutionMode.MAP; - } - if (execution == null) { - execution = ExecutionMode.GLOBAL_ORDINALS; - } - assert execution != null; - - DocValueFormat format = config.format(); - if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) { - throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style " - + "include/exclude settings as they can only be applied to string fields. Use an array of values for " - + "include/exclude clauses"); - } - - return execution.create(name, factories, valuesSource, format, bucketCountThresholds, includeExclude, searchContext, parent, - significanceHeuristic, this, pipelineAggregators, metaData); - } - - if ((includeExclude != null) && (includeExclude.isRegexBased())) { - throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style include/exclude " - + "settings as they can only be applied to string fields. Use an array of numeric values for include/exclude clauses " - + "used to filter numeric fields"); - } - - if (valuesSource instanceof ValuesSource.Numeric) { - - if (((ValuesSource.Numeric) valuesSource).isFloatingPoint()) { - throw new UnsupportedOperationException("No support for examining floating point numerics"); - } - IncludeExclude.LongFilter longFilter = null; - if (includeExclude != null) { - longFilter = includeExclude.convertToLongFilter(config.format()); - } - return new SignificantLongTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, config.format(), - bucketCountThresholds, searchContext, parent, significanceHeuristic, this, longFilter, pipelineAggregators, - metaData); - } + SignificantTermsAggregatorSupplier sigTermsAggregatorSupplier = (SignificantTermsAggregatorSupplier) aggregatorSupplier; - throw new AggregationExecutionException("significant_terms aggregation cannot be applied to field [" - + config.fieldContext().field() + "]. It can only be applied to numeric or string fields."); + // TODO we should refactor so that we don't need to use this Factory as a singleton (e.g. stop passing `this` to the aggregators) + return sigTermsAggregatorSupplier.build(name, factories, valuesSource, config.format(), + bucketCountThresholds, includeExclude, executionHint, searchContext, parent, + significanceHeuristic, this, pipelineAggregators, metaData); } public enum ExecutionMode { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorSupplier.java new file mode 100644 index 0000000000000..e9c494ffd1b48 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorSupplier.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.significant; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; +import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +interface SignificantTermsAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + TermsAggregator.BucketCountThresholds bucketCountThresholds, + IncludeExclude includeExclude, + String executionHint, + SearchContext context, + Aggregator parent, + SignificanceHeuristic significanceHeuristic, + SignificantTermsAggregatorFactory sigTermsFactory, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java index d4a9dda43c2ac..c6c1ccce246cc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java @@ -29,18 +29,17 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; import java.util.Objects; -public class RareTermsAggregationBuilder extends ValuesSourceAggregationBuilder { +public class RareTermsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "rare_terms"; private static final ParseField MAX_DOC_COUNT_FIELD_NAME = new ParseField("max_doc_count"); @@ -48,9 +47,9 @@ public class RareTermsAggregationBuilder extends ValuesSourceAggregationBuilder< private static final int MAX_MAX_DOC_COUNT = 100; public static final ObjectParser PARSER = - ObjectParser.fromBuilder(NAME, name -> new RareTermsAggregationBuilder(name, null)); + ObjectParser.fromBuilder(NAME, RareTermsAggregationBuilder::new); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); PARSER.declareLong(RareTermsAggregationBuilder::maxDocCount, MAX_DOC_COUNT_FIELD_NAME); PARSER.declareField((b, v) -> b.includeExclude(IncludeExclude.merge(v, b.includeExclude())), @@ -62,12 +61,16 @@ public class RareTermsAggregationBuilder extends ValuesSourceAggregationBuilder< PARSER.declareDouble(RareTermsAggregationBuilder::setPrecision, PRECISION); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + RareTermsAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + private IncludeExclude includeExclude = null; private int maxDocCount = 1; private double precision = 0.001; - public RareTermsAggregationBuilder(String name, ValueType valueType) { - super(name, CoreValuesSourceType.ANY, valueType); + public RareTermsAggregationBuilder(String name) { + super(name); } private RareTermsAggregationBuilder(RareTermsAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -75,6 +78,11 @@ private RareTermsAggregationBuilder(RareTermsAggregationBuilder clone, Builder f this.includeExclude = clone.includeExclude; } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new RareTermsAggregationBuilder(this, factoriesBuilder, metaData); @@ -84,7 +92,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return new RareTermsAggregatorFactory(name, config, includeExclude, queryShardContext, parent, subFactoriesBuilder, metaData, maxDocCount, precision); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorFactory.java index e30868b34403f..c959baad7bcd4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorFactory.java @@ -30,21 +30,106 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.NonCollectingAggregator; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -public class RareTermsAggregatorFactory extends ValuesSourceAggregatorFactory { +public class RareTermsAggregatorFactory extends ValuesSourceAggregatorFactory { private final IncludeExclude includeExclude; private final int maxDocCount; private final double precision; - RareTermsAggregatorFactory(String name, ValuesSourceConfig config, + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(RareTermsAggregationBuilder.NAME, + List.of(CoreValuesSourceType.BYTES, CoreValuesSourceType.IP), + RareTermsAggregatorFactory.bytesSupplier()); + + valuesSourceRegistry.register(RareTermsAggregationBuilder.NAME, + List.of(CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN, CoreValuesSourceType.NUMERIC), + RareTermsAggregatorFactory.numericSupplier()); + } + + /** + * This supplier is used for all the field types that should be aggregated as bytes/strings, + * including those that need global ordinals + */ + private static RareTermsAggregatorSupplier bytesSupplier() { + return new RareTermsAggregatorSupplier() { + @Override + public Aggregator build(String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + int maxDocCount, + double precision, + IncludeExclude includeExclude, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException { + + ExecutionMode execution = ExecutionMode.MAP; //TODO global ords not implemented yet, only supports "map" + + if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) { + throw new IllegalArgumentException("Aggregation [" + name + "] cannot support " + + "regular expression style include/exclude settings as they can only be applied to string fields. " + + "Use an array of values for include/exclude clauses"); + } + + return execution.create(name, factories, valuesSource, format, + includeExclude, context, parent, pipelineAggregators, metaData, maxDocCount, precision); + + } + }; + } + + /** + * This supplier is used for all fields that expect to be aggregated as a numeric value. + * This includes floating points, and formatted types that use numerics internally for storage (date, boolean, etc) + */ + private static RareTermsAggregatorSupplier numericSupplier() { + return new RareTermsAggregatorSupplier() { + @Override + public Aggregator build(String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + int maxDocCount, + double precision, + IncludeExclude includeExclude, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException { + + if ((includeExclude != null) && (includeExclude.isRegexBased())) { + throw new IllegalArgumentException("Aggregation [" + name + "] cannot support regular expression " + + "style include/exclude settings as they can only be applied to string fields. Use an array of numeric " + + "values for include/exclude clauses used to filter numeric fields"); + } + + IncludeExclude.LongFilter longFilter = null; + if (((ValuesSource.Numeric) valuesSource).isFloatingPoint()) { + throw new IllegalArgumentException("RareTerms aggregation does not support floating point fields."); + } + if (includeExclude != null) { + longFilter = includeExclude.convertToLongFilter(format); + } + return new LongRareTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, format, + context, parent, longFilter, maxDocCount, precision, pipelineAggregators, metaData); + } + }; + } + + RareTermsAggregatorFactory(String name, ValuesSourceConfig config, IncludeExclude includeExclude, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, @@ -79,40 +164,16 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - if (valuesSource instanceof ValuesSource.Bytes) { - ExecutionMode execution = ExecutionMode.MAP; //TODO global ords not implemented yet, only supports "map" - - DocValueFormat format = config.format(); - if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) { - throw new AggregationExecutionException("Aggregation [" + name + "] cannot support " + - "regular expression style include/exclude settings as they can only be applied to string fields. " + - "Use an array of values for include/exclude clauses"); - } - - return execution.create(name, factories, valuesSource, format, - includeExclude, searchContext, parent, pipelineAggregators, metaData, maxDocCount, precision); - } - - if ((includeExclude != null) && (includeExclude.isRegexBased())) { - throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style include/exclude " - + "settings as they can only be applied to string fields. Use an array of numeric values for include/exclude clauses " + - "used to filter numeric fields"); - } - if (valuesSource instanceof ValuesSource.Numeric) { - IncludeExclude.LongFilter longFilter = null; - if (((ValuesSource.Numeric) valuesSource).isFloatingPoint()) { - throw new AggregationExecutionException("RareTerms aggregation does not support floating point fields."); - } - if (includeExclude != null) { - longFilter = includeExclude.convertToLongFilter(config.format()); - } - return new LongRareTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, config.format(), - searchContext, parent, longFilter, maxDocCount, precision, pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + RareTermsAggregationBuilder.NAME); + if (aggregatorSupplier instanceof RareTermsAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected RareTermsAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); } - throw new AggregationExecutionException("RareTerms aggregation cannot be applied to field [" + config.fieldContext().field() - + "]. It can only be applied to numeric or string fields."); + return ((RareTermsAggregatorSupplier) aggregatorSupplier).build(name, factories, valuesSource, config.format(), + maxDocCount, precision, includeExclude, searchContext, parent, pipelineAggregators, metaData); } public enum ExecutionMode { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorSupplier.java new file mode 100644 index 0000000000000..98e718d6e3b10 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorSupplier.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.terms; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +interface RareTermsAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + int maxDocCount, + double precision, + IncludeExclude includeExclude, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java index f57785404f148..18a902f7a4dc1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java @@ -37,19 +37,18 @@ import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Objects; -public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder +public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder implements MultiBucketAggregationBuilder { public static final String NAME = "terms"; @@ -65,9 +64,9 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder PARSER = - ObjectParser.fromBuilder(NAME, name -> new TermsAggregationBuilder(name, null)); + ObjectParser.fromBuilder(NAME, TermsAggregationBuilder::new); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); PARSER.declareBoolean(TermsAggregationBuilder::showTermDocCountError, TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR); @@ -96,6 +95,10 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder metaData) { @@ -118,6 +121,11 @@ protected TermsAggregationBuilder(TermsAggregationBuilder clone, Builder factori this.showTermDocCountError = clone.showTermDocCountError; } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new TermsAggregationBuilder(this, factoriesBuilder, metaData); @@ -127,7 +135,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return new TermsAggregatorFactory(name, config, order, includeExclude, executionHint, collectMode, bucketCountThresholds, showTermDocCountError, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java index d6dea34f35d83..4cf73196f23e6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java @@ -38,16 +38,19 @@ import org.elasticsearch.search.aggregations.bucket.BucketUtils; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory { +public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(TermsAggregatorFactory.class)); static Boolean REMAP_GLOBAL_ORDS, COLLECT_SEGMENT_ORDS; @@ -59,18 +62,141 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory pipelineAggregators, + Map metaData) throws IOException { + + ExecutionMode execution = null; + if (executionHint != null) { + execution = ExecutionMode.fromString(executionHint, deprecationLogger); + } + // In some cases, using ordinals is just not supported: override it + if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals == false) { + execution = ExecutionMode.MAP; + } + final long maxOrd = execution == ExecutionMode.GLOBAL_ORDINALS ? getMaxOrd(valuesSource, context.searcher()) : -1; + if (execution == null) { + execution = ExecutionMode.GLOBAL_ORDINALS; + } + if (subAggCollectMode == null) { + subAggCollectMode = SubAggCollectionMode.DEPTH_FIRST; + // TODO can we remove concept of AggregatorFactories.EMPTY? + if (factories != AggregatorFactories.EMPTY) { + subAggCollectMode = subAggCollectionMode(bucketCountThresholds.getShardSize(), maxOrd); + } + } + + if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) { + // TODO this exception message is not really accurate for the string case. It's really disallowing regex + formatter + throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style " + + "include/exclude settings as they can only be applied to string fields. Use an array of values for " + + "include/exclude clauses"); + } + + // TODO: [Zach] we might want refactor and remove ExecutionMode#create(), moving that logic outside the enum + return execution.create(name, factories, valuesSource, order, format, bucketCountThresholds, includeExclude, + context, parent, subAggCollectMode, showTermDocCountError, pipelineAggregators, metaData); + + } + }; + } + + /** + * This supplier is used for all fields that expect to be aggregated as a numeric value. + * This includes floating points, and formatted types that use numerics internally for storage (date, boolean, etc) + */ + private static TermsAggregatorSupplier numericSupplier() { + return new TermsAggregatorSupplier() { + @Override + public Aggregator build(String name, + AggregatorFactories factories, + ValuesSource valuesSource, + BucketOrder order, + DocValueFormat format, + TermsAggregator.BucketCountThresholds bucketCountThresholds, + IncludeExclude includeExclude, + String executionHint, + SearchContext context, + Aggregator parent, + SubAggCollectionMode subAggCollectMode, + boolean showTermDocCountError, + List pipelineAggregators, + Map metaData) throws IOException { + + if ((includeExclude != null) && (includeExclude.isRegexBased())) { + throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style " + + "include/exclude settings as they can only be applied to string fields. Use an array of numeric values for " + + "include/exclude clauses used to filter numeric fields"); + } + + IncludeExclude.LongFilter longFilter = null; + if (subAggCollectMode == null) { + // TODO can we remove concept of AggregatorFactories.EMPTY? + if (factories != AggregatorFactories.EMPTY) { + subAggCollectMode = subAggCollectionMode(bucketCountThresholds.getShardSize(), -1); + } else { + subAggCollectMode = SubAggCollectionMode.DEPTH_FIRST; + } + } + if (((ValuesSource.Numeric) valuesSource).isFloatingPoint()) { + if (includeExclude != null) { + longFilter = includeExclude.convertToDoubleFilter(); + } + return new DoubleTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, format, order, + bucketCountThresholds, context, parent, subAggCollectMode, showTermDocCountError, longFilter, + pipelineAggregators, metaData); + } + if (includeExclude != null) { + longFilter = includeExclude.convertToLongFilter(format); + } + return new LongTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, format, order, + bucketCountThresholds, context, parent, subAggCollectMode, showTermDocCountError, longFilter, + pipelineAggregators, metaData); + } + }; + } + TermsAggregatorFactory(String name, - ValuesSourceConfig config, - BucketOrder order, - IncludeExclude includeExclude, - String executionHint, - SubAggCollectionMode collectMode, - TermsAggregator.BucketCountThresholds bucketCountThresholds, - boolean showTermDocCountError, - QueryShardContext queryShardContext, - AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder, - Map metaData) throws IOException { + ValuesSourceConfig config, + BucketOrder order, + IncludeExclude includeExclude, + String executionHint, + SubAggCollectionMode collectMode, + TermsAggregator.BucketCountThresholds bucketCountThresholds, + boolean showTermDocCountError, + QueryShardContext queryShardContext, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); this.order = order; this.includeExclude = includeExclude; @@ -111,89 +237,36 @@ private static boolean isAggregationSort(BucketOrder order) { @Override protected Aggregator doCreateInternal(ValuesSource valuesSource, - SearchContext searchContext, - Aggregator parent, - boolean collectsFromSingleBucket, - List pipelineAggregators, - Map metaData) throws IOException { + SearchContext searchContext, + Aggregator parent, + boolean collectsFromSingleBucket, + List pipelineAggregators, + Map metaData) throws IOException { if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } + + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + TermsAggregationBuilder.NAME); + if (aggregatorSupplier instanceof TermsAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected TermsAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + + TermsAggregatorSupplier termsAggregatorSupplier = (TermsAggregatorSupplier) aggregatorSupplier; BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds); if (InternalOrder.isKeyOrder(order) == false - && bucketCountThresholds.getShardSize() == TermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) { + && bucketCountThresholds.getShardSize() == TermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) { // The user has not made a shardSize selection. Use default // heuristic to avoid any wrong-ranking caused by distributed // counting bucketCountThresholds.setShardSize(BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize())); } bucketCountThresholds.ensureValidity(); - if (valuesSource instanceof ValuesSource.Bytes) { - ExecutionMode execution = null; - if (executionHint != null) { - execution = ExecutionMode.fromString(executionHint, deprecationLogger); - } - // In some cases, using ordinals is just not supported: override it - if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals == false) { - execution = ExecutionMode.MAP; - } - final long maxOrd = execution == ExecutionMode.GLOBAL_ORDINALS ? getMaxOrd(valuesSource, searchContext.searcher()) : -1; - if (execution == null) { - execution = ExecutionMode.GLOBAL_ORDINALS; - } - SubAggCollectionMode cm = collectMode; - if (cm == null) { - cm = SubAggCollectionMode.DEPTH_FIRST; - if (factories != AggregatorFactories.EMPTY) { - cm = subAggCollectionMode(bucketCountThresholds.getShardSize(), maxOrd); - } - } - - DocValueFormat format = config.format(); - if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) { - throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style " - + "include/exclude settings as they can only be applied to string fields. Use an array of values for " - + "include/exclude clauses"); - } - - return execution.create(name, factories, valuesSource, order, format, - bucketCountThresholds, includeExclude, searchContext, parent, cm, showTermDocCountError, pipelineAggregators, metaData); - } - - if ((includeExclude != null) && (includeExclude.isRegexBased())) { - throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style " - + "include/exclude settings as they can only be applied to string fields. Use an array of numeric values for " - + "include/exclude clauses used to filter numeric fields"); - } - - if (valuesSource instanceof ValuesSource.Numeric) { - IncludeExclude.LongFilter longFilter = null; - SubAggCollectionMode cm = collectMode; - if (cm == null) { - if (factories != AggregatorFactories.EMPTY) { - cm = subAggCollectionMode(bucketCountThresholds.getShardSize(), -1); - } else { - cm = SubAggCollectionMode.DEPTH_FIRST; - } - } - if (((ValuesSource.Numeric) valuesSource).isFloatingPoint()) { - if (includeExclude != null) { - longFilter = includeExclude.convertToDoubleFilter(); - } - return new DoubleTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, config.format(), order, - bucketCountThresholds, searchContext, parent, cm, showTermDocCountError, longFilter, - pipelineAggregators, metaData); - } - if (includeExclude != null) { - longFilter = includeExclude.convertToLongFilter(config.format()); - } - return new LongTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, config.format(), order, - bucketCountThresholds, searchContext, parent, cm, showTermDocCountError, longFilter, pipelineAggregators, - metaData); - } - throw new AggregationExecutionException("terms aggregation cannot be applied to field [" + config.fieldContext().field() - + "]. It can only be applied to numeric or string fields."); + return termsAggregatorSupplier.build(name, factories, valuesSource, order, config.format(), + bucketCountThresholds, includeExclude, executionHint, searchContext, parent, collectMode, + showTermDocCountError, pipelineAggregators, metaData); } // return the SubAggCollectionMode that this aggregation should use based on the expected size @@ -214,7 +287,7 @@ static SubAggCollectionMode subAggCollectionMode(int expectedSize, long maxOrd) * Get the maximum global ordinal value for the provided {@link ValuesSource} or -1 * if the values source is not an instance of {@link ValuesSource.Bytes.WithOrdinals}. */ - static long getMaxOrd(ValuesSource source, IndexSearcher searcher) throws IOException { + private static long getMaxOrd(ValuesSource source, IndexSearcher searcher) throws IOException { if (source instanceof ValuesSource.Bytes.WithOrdinals) { ValuesSource.Bytes.WithOrdinals valueSourceWithOrdinals = (ValuesSource.Bytes.WithOrdinals) source; return valueSourceWithOrdinals.globalMaxOrd(searcher); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorSupplier.java new file mode 100644 index 0000000000000..ccfb082ac98ce --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorSupplier.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.terms; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +interface TermsAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + AggregatorFactories factories, + ValuesSource valuesSource, + BucketOrder order, + DocValueFormat format, + TermsAggregator.BucketCountThresholds bucketCountThresholds, + IncludeExclude includeExclude, + String executionHint, + SearchContext context, + Aggregator parent, + Aggregator.SubAggCollectionMode subAggCollectMode, + boolean showTermDocCountError, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java index 5d9e616a39718..546db9d447485 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java @@ -26,8 +26,6 @@ import org.elasticsearch.common.util.ArrayUtils; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ObjectArray; -import org.elasticsearch.index.fielddata.HistogramValue; -import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; @@ -78,18 +76,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - if (valuesSource instanceof ValuesSource.Histogram) { - final HistogramValues values = ((ValuesSource.Histogram)valuesSource).getHistogramValues(ctx); - return collectHistogramValues(values, bigArrays, sub); - } else { - final SortedNumericDoubleValues values = ((ValuesSource.Numeric)valuesSource).doubleValues(ctx); - return collectNumeric(values, bigArrays, sub); - } - - } - private LeafBucketCollector collectNumeric(final SortedNumericDoubleValues values, - final BigArrays bigArrays, final LeafBucketCollector sub) { + final SortedNumericDoubleValues values = ((ValuesSource.Numeric)valuesSource).doubleValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -102,22 +90,7 @@ public void collect(int doc, long bucket) throws IOException { } } }; - } - private LeafBucketCollector collectHistogramValues(final HistogramValues values, - final BigArrays bigArrays, final LeafBucketCollector sub) { - return new LeafBucketCollectorBase(sub, values) { - @Override - public void collect(int doc, long bucket) throws IOException { - DoubleHistogram state = getExistingOrNewHistogram(bigArrays, bucket); - if (values.advanceExact(doc)) { - final HistogramValue sketch = values.histogram(); - while (sketch.next()) { - state.recordValueWithCount(sketch.value(), sketch.count()); - } - } - } - }; } private DoubleHistogram getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) { @@ -125,12 +98,13 @@ private DoubleHistogram getExistingOrNewHistogram(final BigArrays bigArrays, lon DoubleHistogram state = states.get(bucket); if (state == null) { state = new DoubleHistogram(numberOfSignificantValueDigits); - // Set the histogram to autosize so it can resize itself as - // the data range increases. Resize operations should be - // rare as the histogram buckets are exponential (on the top - // level). In the future we could expose the range as an - // option on the request so the histogram can be fixed at - // initialisation and doesn't need resizing. + /* Set the histogram to autosize so it can resize itself as + the data range increases. Resize operations should be + rare as the histogram buckets are exponential (on the top + level). In the future we could expose the range as an + option on the request so the histogram can be fixed at + initialisation and doesn't need resizing. + */ state.setAutoResize(true); states.set(bucket, state); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java index 8612f0781c1ee..b9cbc3474a04d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java @@ -28,11 +28,8 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; import java.io.IOException; import java.util.Arrays; @@ -107,7 +104,7 @@ public static > ConstructingO return ctor.apply(name, values, percentilesConfig); }); - ValuesSourceParserHelper.declareAnyFields(parser, true, true); + ValuesSourceAggregationBuilder.declareFields(parser, true, true, false); parser.declareDoubleArray(ConstructingObjectParser.optionalConstructorArg(), valuesField); parser.declareBoolean(T::keyed, KEYED_FIELD); parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), PercentilesMethod.TDIGEST_PARSER, @@ -120,7 +117,7 @@ public static > ConstructingO AbstractPercentilesAggregationBuilder(String name, double[] values, PercentilesConfig percentilesConfig, ParseField valuesField) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); if (values == null) { throw new IllegalArgumentException("[" + valuesField.getPreferredName() + "] must not be null: [" + name + "]"); } @@ -144,7 +141,7 @@ public static > ConstructingO } AbstractPercentilesAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(in); values = in.readDoubleArray(); keyed = in.readBoolean(); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java index 1b78db480068e..4823bd805fb17 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java @@ -25,8 +25,6 @@ import org.elasticsearch.common.util.ArrayUtils; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ObjectArray; -import org.elasticsearch.index.fielddata.HistogramValue; -import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; @@ -77,18 +75,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - if (valuesSource instanceof ValuesSource.Histogram) { - final HistogramValues values = ((ValuesSource.Histogram)valuesSource).getHistogramValues(ctx); - return collectHistogramValues(values, bigArrays, sub); - } else { - final SortedNumericDoubleValues values = ((ValuesSource.Numeric)valuesSource).doubleValues(ctx); - return collectNumeric(values, bigArrays, sub); - } - - } - - private LeafBucketCollector collectNumeric(final SortedNumericDoubleValues values, - final BigArrays bigArrays, final LeafBucketCollector sub) { + final SortedNumericDoubleValues values = ((ValuesSource.Numeric)valuesSource).doubleValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -103,22 +90,6 @@ public void collect(int doc, long bucket) throws IOException { }; } - private LeafBucketCollector collectHistogramValues(final HistogramValues values, - final BigArrays bigArrays, final LeafBucketCollector sub) { - return new LeafBucketCollectorBase(sub, values) { - @Override - public void collect(int doc, long bucket) throws IOException { - TDigestState state = getExistingOrNewHistogram(bigArrays, bucket); - if (values.advanceExact(doc)) { - final HistogramValue sketch = values.histogram(); - while(sketch.next()) { - state.add(sketch.value(), sketch.count()); - } - } - } - }; - } - private TDigestState getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) { states = bigArrays.grow(states, bucket + 1); TDigestState state = states.get(bucket); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java index 54f7c0c8e78f0..be747ab1067ec 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java @@ -28,12 +28,11 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -43,22 +42,31 @@ public class AvgAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, AvgAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); + } + + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + AvgAggregatorFactory.registerAggregators(valuesSourceRegistry); } public AvgAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); } public AvgAggregationBuilder(AvgAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.NUMERIC; + } + /** * Read from a stream. */ public AvgAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(in); } @Override @@ -72,7 +80,7 @@ protected void innerWriteTo(StreamOutput out) { } @Override - protected AvgAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, + protected AvgAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new AvgAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java index 39a6c40aace71..a510665bf7ab5 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java @@ -20,27 +20,49 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class AvgAggregatorFactory extends ValuesSourceAggregatorFactory { +class AvgAggregatorFactory extends ValuesSourceAggregatorFactory { - AvgAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, - AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { + AvgAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, + AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(AvgAggregationBuilder.NAME, + List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + new MetricAggregatorSupplier() { + @Override + public Aggregator build(String name, + ValuesSource valuesSource, + DocValueFormat formatter, + SearchContext context, + Aggregator parent, + List pipelineAggregators, Map metaData) throws IOException { + return new AvgAggregator(name, (Numeric) valuesSource, formatter, context, parent, pipelineAggregators, metaData); + } + }); + } + @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, @@ -50,12 +72,20 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(Numeric valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new AvgAggregator(name, valuesSource, config.format(), searchContext, parent, pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + AvgAggregationBuilder.NAME); + + if (aggregatorSupplier instanceof MetricAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected MetricAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((MetricAggregatorSupplier) aggregatorSupplier).build(name, valuesSource, config.format(), searchContext, parent, + pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java index 8c9e23f822111..f67e8d3fe50d4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java @@ -30,11 +30,11 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -49,17 +49,21 @@ public final class CardinalityAggregationBuilder public static final ParseField PRECISION_THRESHOLD_FIELD = new ParseField("precision_threshold"); public static final ObjectParser PARSER = - ObjectParser.fromBuilder(NAME, name -> new CardinalityAggregationBuilder(name, null)); + ObjectParser.fromBuilder(NAME, CardinalityAggregationBuilder::new); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, false, false); PARSER.declareLong(CardinalityAggregationBuilder::precisionThreshold, CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD); PARSER.declareLong((b, v) -> {/*ignore*/}, REHASH); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + CardinalityAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + private Long precisionThreshold = null; - public CardinalityAggregationBuilder(String name, ValueType targetValueType) { - super(name, CoreValuesSourceType.ANY, targetValueType); + public CardinalityAggregationBuilder(String name) { + super(name); } public CardinalityAggregationBuilder(CardinalityAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -67,11 +71,16 @@ public CardinalityAggregationBuilder(CardinalityAggregationBuilder clone, Builde this.precisionThreshold = clone.precisionThreshold; } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + /** * Read from a stream. */ public CardinalityAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.ANY); + super(in); if (in.readBoolean()) { precisionThreshold = in.readLong(); } @@ -119,7 +128,7 @@ public Long precisionThreshold() { } @Override - protected CardinalityAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, + protected CardinalityAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new CardinalityAggregatorFactory(name, config, precisionThreshold, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java index 4fa4a8dddac9c..63da48e944d86 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java @@ -20,24 +20,27 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory { +class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory { private final Long precisionThreshold; - CardinalityAggregatorFactory(String name, ValuesSourceConfig config, + CardinalityAggregatorFactory(String name, ValuesSourceConfig config, Long precisionThreshold, QueryShardContext queryShardContext, AggregatorFactory parent, @@ -47,6 +50,25 @@ class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory pipelineAggregators, + Map metaData) throws IOException { + return new CardinalityAggregator(name, valuesSource, precision, context, parent, pipelineAggregators, metaData); + } + }; + } + @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, @@ -62,8 +84,14 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new CardinalityAggregator(name, valuesSource, precision(), searchContext, parent, pipelineAggregators, - metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + CardinalityAggregationBuilder.NAME); + if (aggregatorSupplier instanceof CardinalityAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected CardinalityAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + CardinalityAggregatorSupplier cardinalityAggregatorSupplier = (CardinalityAggregatorSupplier) aggregatorSupplier; + return cardinalityAggregatorSupplier.build(name, valuesSource, precision(), searchContext, parent, pipelineAggregators, metaData); } private int precision() { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorSupplier.java new file mode 100644 index 0000000000000..a27ae4d54c935 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorSupplier.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface CardinalityAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + ValuesSource valuesSource, + int precision, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException; + +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java index bc246e22c7a73..94d9ec9b8931a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java @@ -28,12 +28,10 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -46,14 +44,14 @@ public class ExtendedStatsAggregationBuilder public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, ExtendedStatsAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); PARSER.declareDouble(ExtendedStatsAggregationBuilder::sigma, ExtendedStatsAggregator.SIGMA_FIELD); } private double sigma = 2.0; public ExtendedStatsAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); } protected ExtendedStatsAggregationBuilder(ExtendedStatsAggregationBuilder clone, @@ -71,10 +69,15 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map config, + protected ExtendedStatsAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new ExtendedStatsAggregatorFactory(name, config, sigma, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java index 527c6093aa80b..cba2e1cf72eb2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -34,12 +35,12 @@ import java.util.List; import java.util.Map; -class ExtendedStatsAggregatorFactory extends ValuesSourceAggregatorFactory { +class ExtendedStatsAggregatorFactory extends ValuesSourceAggregatorFactory { private final double sigma; ExtendedStatsAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, double sigma, QueryShardContext queryShardContext, AggregatorFactory parent, @@ -59,13 +60,17 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(Numeric valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new ExtendedStatsAggregator(name, valuesSource, config.format(), searchContext, + if (valuesSource instanceof Numeric == false) { + throw new AggregationExecutionException("ValuesSource type " + valuesSource.toString() + "is not supported for aggregation " + + this.name()); + } + return new ExtendedStatsAggregator(name, (Numeric) valuesSource, config.format(), searchContext, parent, sigma, pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java index 80658243a0794..e7e836f547729 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java @@ -32,26 +32,33 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; -public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder { +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "geo_bounds"; public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, GeoBoundsAggregationBuilder::new); static { - ValuesSourceParserHelper.declareGeoFields(PARSER, false, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, false, false, false); PARSER.declareBoolean(GeoBoundsAggregationBuilder::wrapLongitude, GeoBoundsAggregator.WRAP_LONGITUDE_FIELD); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + GeoBoundsAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + private boolean wrapLongitude = true; public GeoBoundsAggregationBuilder(String name) { - super(name, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(name); } protected GeoBoundsAggregationBuilder(GeoBoundsAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -68,7 +75,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map config, + protected GeoBoundsAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new GeoBoundsAggregatorFactory(name, config, wrapLongitude, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java index 462aff381d3d8..375fd050cfb39 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java @@ -20,25 +20,29 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory { +class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory { private final boolean wrapLongitude; GeoBoundsAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, boolean wrapLongitude, QueryShardContext queryShardContext, AggregatorFactory parent, @@ -57,12 +61,28 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(ValuesSource.GeoPoint valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new GeoBoundsAggregator(name, searchContext, parent, valuesSource, wrapLongitude, pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry() + .getAggregator(config.valueSourceType(), GeoBoundsAggregationBuilder.NAME); + + if (aggregatorSupplier instanceof GeoBoundsAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected " + + GeoBoundsAggregatorSupplier.class.getName() + ", found [" + aggregatorSupplier.getClass().toString() + "]"); + } + + return ((GeoBoundsAggregatorSupplier) aggregatorSupplier).build(name, searchContext, parent, valuesSource, wrapLongitude, + pipelineAggregators, metaData); + } + + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(GeoBoundsAggregationBuilder.NAME, CoreValuesSourceType.GEOPOINT, + (GeoBoundsAggregatorSupplier) (name, aggregationContext, parent, valuesSource, wrapLongitude, pipelineAggregators, metaData) + -> new GeoBoundsAggregator(name, aggregationContext, parent, (ValuesSource.GeoPoint) valuesSource, + wrapLongitude, pipelineAggregators, metaData)); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorSupplier.java new file mode 100644 index 0000000000000..4061f2022eb83 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorSupplier.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@FunctionalInterface +public interface GeoBoundsAggregatorSupplier extends AggregatorSupplier { + + GeoBoundsAggregator build(String name, SearchContext aggregationContext, Aggregator parent, + ValuesSource valuesSource, boolean wrapLongitude, List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java index afa0329433a6e..c6fbe1dd2958b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java @@ -19,9 +19,6 @@ package org.elasticsearch.search.aggregations.metrics; -import java.io.IOException; -import java.util.Map; - import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; @@ -31,11 +28,14 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + +import java.io.IOException; +import java.util.Map; public class GeoCentroidAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly { @@ -44,17 +44,26 @@ public class GeoCentroidAggregationBuilder public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, GeoCentroidAggregationBuilder::new); static { - ValuesSourceParserHelper.declareGeoFields(PARSER, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, false, false); + } + + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + GeoCentroidAggregatorFactory.registerAggregators(valuesSourceRegistry); } public GeoCentroidAggregationBuilder(String name) { - super(name, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(name); } protected GeoCentroidAggregationBuilder(GeoCentroidAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.GEOPOINT; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new GeoCentroidAggregationBuilder(this, factoriesBuilder, metaData); @@ -64,7 +73,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map config, + protected GeoCentroidAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new GeoCentroidAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java index 73200f18d91af..8a109b6ebeb46 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java @@ -20,23 +20,27 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory { +class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory { GeoCentroidAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, @@ -53,12 +57,26 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(ValuesSource.GeoPoint valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new GeoCentroidAggregator(name, searchContext, parent, valuesSource, pipelineAggregators, metaData); + + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + GeoCentroidAggregationBuilder.NAME); + if (aggregatorSupplier instanceof GeoCentroidAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected " + + GeoCentroidAggregatorSupplier.class.getName() + ", found [" + aggregatorSupplier.getClass().toString() + "]"); + } + return ((GeoCentroidAggregatorSupplier) aggregatorSupplier).build(name, searchContext, parent, valuesSource, + pipelineAggregators, metaData); + } + + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(GeoCentroidAggregationBuilder.NAME, CoreValuesSourceType.GEOPOINT, + (GeoCentroidAggregatorSupplier) (name, context, parent, valuesSource, pipelineAggregators, metaData) -> + new GeoCentroidAggregator(name, context, parent, (ValuesSource.GeoPoint) valuesSource, pipelineAggregators, metaData)); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorSupplier.java new file mode 100644 index 0000000000000..7c1d34ae20977 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorSupplier.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@FunctionalInterface +public interface GeoCentroidAggregatorSupplier extends AggregatorSupplier { + + GeoCentroidAggregator build(String name, SearchContext context, Aggregator parent, + ValuesSource valuesSource, List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java new file mode 100644 index 0000000000000..4b08afbc5977e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@FunctionalInterface +public interface GeoGridAggregatorSupplier extends AggregatorSupplier { + + GeoGridAggregator build(String name, AggregatorFactories factories, ValuesSource valuesSource, + int precision, GeoBoundingBox geoBoundingBox, int requiredSize, int shardSize, + SearchContext aggregationContext, Aggregator parent, List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java index bfe483d0e3c47..2d684e9b8e637 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java @@ -31,8 +31,8 @@ public class InternalHDRPercentileRanks extends AbstractInternalHDRPercentiles implements PercentileRanks { public static final String NAME = "hdr_percentile_ranks"; - InternalHDRPercentileRanks(String name, double[] cdfValues, DoubleHistogram state, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, Map metaData) { + public InternalHDRPercentileRanks(String name, double[] cdfValues, DoubleHistogram state, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) { super(name, cdfValues, state, keyed, formatter, pipelineAggregators, metaData); } @@ -74,7 +74,7 @@ protected AbstractInternalHDRPercentiles createReduced(String name, double[] key return new InternalHDRPercentileRanks(name, keys, merged, keyed, format, pipelineAggregators, metaData); } - static double percentileRank(DoubleHistogram state, double value) { + public static double percentileRank(DoubleHistogram state, double value) { if (state.getTotalCount() == 0) { return Double.NaN; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java index 5a62de8a964ec..6f47169e7ed6c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java @@ -31,8 +31,8 @@ public class InternalHDRPercentiles extends AbstractInternalHDRPercentiles implements Percentiles { public static final String NAME = "hdr_percentiles"; - InternalHDRPercentiles(String name, double[] percents, DoubleHistogram state, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, Map metaData) { + public InternalHDRPercentiles(String name, double[] percents, DoubleHistogram state, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) { super(name, percents, state, keyed, formatter, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java index aa82ac5ba6add..69cc407efa032 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java @@ -30,8 +30,8 @@ public class InternalTDigestPercentileRanks extends AbstractInternalTDigestPercentiles implements PercentileRanks { public static final String NAME = "tdigest_percentile_ranks"; - InternalTDigestPercentileRanks(String name, double[] cdfValues, TDigestState state, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, Map metaData) { + public InternalTDigestPercentileRanks(String name, double[] cdfValues, TDigestState state, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) { super(name, cdfValues, state, keyed, formatter, pipelineAggregators, metaData); } @@ -73,7 +73,7 @@ protected AbstractInternalTDigestPercentiles createReduced(String name, double[] return new InternalTDigestPercentileRanks(name, keys, merged, keyed, format, pipelineAggregators, metaData); } - static double percentileRank(TDigestState state, double value) { + public static double percentileRank(TDigestState state, double value) { double percentileRank = state.cdf(value); if (percentileRank < 0) { percentileRank = 0; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentiles.java index 28f1230bec713..6c858d7a1ec48 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentiles.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentiles.java @@ -30,8 +30,8 @@ public class InternalTDigestPercentiles extends AbstractInternalTDigestPercentiles implements Percentiles { public static final String NAME = "tdigest_percentiles"; - InternalTDigestPercentiles(String name, double[] percents, TDigestState state, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, Map metaData) { + public InternalTDigestPercentiles(String name, double[] percents, TDigestState state, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) { super(name, percents, state, keyed, formatter, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java index 7b31ab22cc7d9..81bf8bf9f63e0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java @@ -28,12 +28,11 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -43,17 +42,26 @@ public class MaxAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, MaxAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); + } + + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + MaxAggregatorFactory.registerAggregators(valuesSourceRegistry); } public MaxAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); } protected MaxAggregationBuilder(MaxAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new MaxAggregationBuilder(this, factoriesBuilder, metaData); @@ -63,7 +71,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map config, + protected MaxAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new MaxAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java index d00bbda46049e..b0a1f89bcb90e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java @@ -59,7 +59,7 @@ class MaxAggregator extends NumericMetricsAggregator.SingleValue { DoubleArray maxes; MaxAggregator(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, ValuesSource.Numeric valuesSource, SearchContext context, Aggregator parent, List pipelineAggregators, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java index b345cca7dc68c..7604460fbc741 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java @@ -20,24 +20,46 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class MaxAggregatorFactory extends ValuesSourceAggregatorFactory { +class MaxAggregatorFactory extends ValuesSourceAggregatorFactory { - MaxAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, - AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(MaxAggregationBuilder.NAME, + List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + new MinMaxAggregatorSupplier() { + @Override + public Aggregator build(String name, + ValuesSourceConfig config, + ValuesSource valuesSource, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException { + return new MaxAggregator(name, config, (Numeric) valuesSource, context, parent, pipelineAggregators, metaData); + } + }); + } + + MaxAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, + AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } @@ -50,12 +72,20 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(Numeric valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new MaxAggregator(name, config, valuesSource, searchContext, parent, pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + MaxAggregationBuilder.NAME); + + if (aggregatorSupplier instanceof MinMaxAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected MinMaxAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((MinMaxAggregatorSupplier) aggregatorSupplier).build(name, config, valuesSource, searchContext, parent, + pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java index 3f35d4bf5a4d1..335d1d42ae3bf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java @@ -29,12 +29,13 @@ import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.LeafOnly; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -49,18 +50,22 @@ public class MedianAbsoluteDeviationAggregationBuilder extends LeafOnly PARSER = ObjectParser.fromBuilder(NAME, MedianAbsoluteDeviationAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); PARSER.declareDouble(MedianAbsoluteDeviationAggregationBuilder::compression, COMPRESSION_FIELD); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + MedianAbsoluteDeviationAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + private double compression = 1000d; public MedianAbsoluteDeviationAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); } public MedianAbsoluteDeviationAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(in); compression = in.readDouble(); } @@ -95,16 +100,21 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu return new MedianAbsoluteDeviationAggregationBuilder(this, factoriesBuilder, metaData); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override protected void innerWriteTo(StreamOutput out) throws IOException { out.writeDouble(compression); } @Override - protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder) + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder) throws IOException { return new MedianAbsoluteDeviationAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData, compression); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java index 1f26b634cc275..9e69c9038f0f9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java @@ -20,40 +20,72 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -public class MedianAbsoluteDeviationAggregatorFactory extends ValuesSourceAggregatorFactory { +public class MedianAbsoluteDeviationAggregatorFactory extends ValuesSourceAggregatorFactory { private final double compression; MedianAbsoluteDeviationAggregatorFactory(String name, - ValuesSourceConfig config, - QueryShardContext queryShardContext, - AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder, - Map metaData, - double compression) throws IOException { + ValuesSourceConfig config, + QueryShardContext queryShardContext, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metaData, + double compression) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); this.compression = compression; } + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(MedianAbsoluteDeviationAggregationBuilder.NAME, + CoreValuesSourceType.NUMERIC, + new MedianAbsoluteDeviationAggregatorSupplier() { + @Override + public Aggregator build(String name, + ValuesSource valuesSource, + DocValueFormat format, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData, + double compression) throws IOException { + return new MedianAbsoluteDeviationAggregator( + name, + context, + parent, + pipelineAggregators, + metaData, + (ValuesSource.Numeric) valuesSource, + format, + compression + ); + } + }); + } + @Override protected Aggregator createUnmapped(SearchContext searchContext, - Aggregator parent, - List pipelineAggregators, - Map metaData) throws IOException { + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException { return new MedianAbsoluteDeviationAggregator( name, @@ -68,22 +100,20 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + MedianAbsoluteDeviationAggregationBuilder.NAME); - return new MedianAbsoluteDeviationAggregator( - name, - searchContext, - parent, - pipelineAggregators, - metaData, - valuesSource, - config.format(), - compression - ); + if (aggregatorSupplier instanceof MedianAbsoluteDeviationAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected MedianAbsoluteDeviationAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((MedianAbsoluteDeviationAggregatorSupplier) aggregatorSupplier).build(name, valuesSource, config.format(), + searchContext, parent, pipelineAggregators, metaData, compression); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorSupplier.java new file mode 100644 index 0000000000000..4451f731487a8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorSupplier.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface MedianAbsoluteDeviationAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + ValuesSource valuesSource, + DocValueFormat format, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData, + double compression) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricAggregatorSupplier.java new file mode 100644 index 0000000000000..4aa2136f03fbc --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricAggregatorSupplier.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface MetricAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + ValuesSource valuesSource, + DocValueFormat format, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java index 90099dac2a967..820764b3f6ec2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java @@ -28,12 +28,11 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -43,11 +42,11 @@ public class MinAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, MinAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); } public MinAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); } protected MinAggregationBuilder(MinAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -59,11 +58,20 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map config, + protected MinAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new MinAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java index b8bfae41b2314..a71fef06f63f5 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java @@ -62,7 +62,7 @@ class MinAggregator extends NumericMetricsAggregator.SingleValue { DoubleArray mins; MinAggregator(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, ValuesSource.Numeric valuesSource, SearchContext context, Aggregator parent, @@ -172,7 +172,7 @@ public void doClose() { * @param config The config for the values source metric. */ static Function getPointReaderOrNull(SearchContext context, Aggregator parent, - ValuesSourceConfig config) { + ValuesSourceConfig config) { if (context.query() != null && context.query().getClass() != MatchAllDocsQuery.class) { return null; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java index 753c7643c206e..b732763e3d4a2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java @@ -20,24 +20,46 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class MinAggregatorFactory extends ValuesSourceAggregatorFactory { +class MinAggregatorFactory extends ValuesSourceAggregatorFactory { - MinAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, - AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(MinAggregationBuilder.NAME, + List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + new MinMaxAggregatorSupplier() { + @Override + public Aggregator build(String name, + ValuesSourceConfig config, + ValuesSource valuesSource, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException { + return new MinAggregator(name, config, (Numeric) valuesSource, context, parent, pipelineAggregators, metaData); + } + }); + } + + MinAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, + AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } @@ -50,12 +72,20 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(Numeric valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new MinAggregator(name, config, valuesSource, searchContext, parent, pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + MinAggregationBuilder.NAME); + + if (aggregatorSupplier instanceof MinMaxAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected MinMaxAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((MinMaxAggregatorSupplier) aggregatorSupplier).build(name, config, valuesSource, searchContext, parent, + pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinMaxAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinMaxAggregatorSupplier.java new file mode 100644 index 0000000000000..51e4c4d95bd1a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinMaxAggregatorSupplier.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface MinMaxAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + ValuesSourceConfig config, + ValuesSource valuesSource, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java index 4a0e87be7bc0e..26ef1dd5a609d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java @@ -26,9 +26,11 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -44,6 +46,10 @@ public class PercentileRanksAggregationBuilder extends AbstractPercentilesAggreg PercentilesConfig.TDigest::new, VALUES_FIELD); + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + PercentileRanksAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + public PercentileRanksAggregationBuilder(String name, double[] values) { this(name, values, null); } @@ -67,6 +73,11 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new PercentileRanksAggregatorFactory(name, config, values, configOrDefault(), keyed, queryShardContext, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java index 99ddd0162bec0..39080f5d6f62e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java @@ -20,27 +20,48 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class PercentileRanksAggregatorFactory extends ValuesSourceAggregatorFactory { +class PercentileRanksAggregatorFactory extends ValuesSourceAggregatorFactory { private final double[] percents; private final PercentilesConfig percentilesConfig; private final boolean keyed; + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(PercentileRanksAggregationBuilder.NAME, + List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + new PercentilesAggregatorSupplier() { + @Override + public Aggregator build(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] percents, PercentilesConfig percentilesConfig, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) throws IOException { + + return percentilesConfig.createPercentileRanksAggregator(name, valuesSource, context, parent, percents, keyed, + formatter, pipelineAggregators, metaData); + } + } + ); + } + PercentileRanksAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, double[] percents, PercentilesConfig percentilesConfig, boolean keyed, @@ -71,7 +92,15 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return percentilesConfig.createPercentileRanksAggregator(name, valuesSource, searchContext, parent, percents, keyed, + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + PercentileRanksAggregationBuilder.NAME); + + if (aggregatorSupplier instanceof PercentilesAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected PercentilesAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + PercentilesAggregatorSupplier percentilesAggregatorSupplier = (PercentilesAggregatorSupplier) aggregatorSupplier; + return percentilesAggregatorSupplier.build(name, valuesSource, searchContext, parent, percents, percentilesConfig, keyed, config.format(), pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java index b5e706c84a39c..101408381bc3c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java @@ -26,9 +26,11 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Arrays; @@ -54,6 +56,10 @@ public class PercentilesAggregationBuilder extends AbstractPercentilesAggregatio PercentilesConfig.TDigest::new, PERCENTS_FIELD); + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + PercentilesAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + public PercentilesAggregationBuilder(StreamInput in) throws IOException { super(in); } @@ -76,6 +82,11 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new PercentilesAggregatorFactory(name, config, values, configOrDefault(), keyed, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java index 570125f3ea1cb..8b3112613e7b7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java @@ -20,13 +20,18 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -37,13 +42,29 @@ * This factory is used to generate both TDigest and HDRHisto aggregators, depending * on the selected method */ -class PercentilesAggregatorFactory extends ValuesSourceAggregatorFactory { +class PercentilesAggregatorFactory extends ValuesSourceAggregatorFactory { private final double[] percents; private final PercentilesConfig percentilesConfig; private final boolean keyed; - PercentilesAggregatorFactory(String name, ValuesSourceConfig config, double[] percents, + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(PercentilesAggregationBuilder.NAME, + List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + new PercentilesAggregatorSupplier() { + @Override + public Aggregator build(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] percents, PercentilesConfig percentilesConfig, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) throws IOException { + + return percentilesConfig.createPercentilesAggregator(name, valuesSource, context, parent, percents, keyed, + formatter, pipelineAggregators, metaData); + } + } + ); + } + + PercentilesAggregatorFactory(String name, ValuesSourceConfig config, double[] percents, PercentilesConfig percentilesConfig, boolean keyed, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { @@ -71,7 +92,15 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, List pipelineAggregators, Map metaData) throws IOException { - return percentilesConfig.createPercentilesAggregator(name, valuesSource, searchContext, parent, percents, keyed, - config.format(), pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + PercentilesAggregationBuilder.NAME); + + if (aggregatorSupplier instanceof PercentilesAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected PercentilesAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + PercentilesAggregatorSupplier percentilesAggregatorSupplier = (PercentilesAggregatorSupplier) aggregatorSupplier; + return percentilesAggregatorSupplier.build(name, valuesSource, searchContext, parent, percents, percentilesConfig, keyed, + config.format(), pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorSupplier.java new file mode 100644 index 0000000000000..782d0b779da78 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorSupplier.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface PercentilesAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + ValuesSource valuesSource, + SearchContext context, + Aggregator parent, + double[] percents, + PercentilesConfig percentilesConfig, + boolean keyed, + DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesConfig.java index af7f9ed5ec412..67af2c18102b1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesConfig.java @@ -69,10 +69,10 @@ public PercentilesMethod getMethod() { return method; } - abstract Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, - double[] values, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, - Map metaData) throws IOException; + public abstract Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] values, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException; abstract Aggregator createPercentileRanksAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, double[] values, boolean keyed, @@ -102,11 +102,11 @@ public static class TDigest extends PercentilesConfig { static final double DEFAULT_COMPRESSION = 100.0; private double compression; - TDigest() { + public TDigest() { this(DEFAULT_COMPRESSION); } - TDigest(double compression) { + public TDigest(double compression) { super(PercentilesMethod.TDIGEST); setCompression(compression); } @@ -128,10 +128,10 @@ public double getCompression() { } @Override - Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, - double[] values, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, - Map metaData) throws IOException { + public Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] values, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException { return new TDigestPercentilesAggregator(name, valuesSource, context, parent, values, compression, keyed, formatter, pipelineAggregators, metaData); } @@ -179,11 +179,11 @@ public static class Hdr extends PercentilesConfig { static final int DEFAULT_NUMBER_SIG_FIGS = 3; private int numberOfSignificantValueDigits; - Hdr() { + public Hdr() { this(DEFAULT_NUMBER_SIG_FIGS); } - Hdr(int numberOfSignificantValueDigits) { + public Hdr(int numberOfSignificantValueDigits) { super(PercentilesMethod.HDR); setNumberOfSignificantValueDigits(numberOfSignificantValueDigits); } @@ -204,10 +204,10 @@ public int getNumberOfSignificantValueDigits() { } @Override - Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, - double[] values, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, - Map metaData) throws IOException { + public Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] values, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException { return new HDRPercentilesAggregator(name, valuesSource, context, parent, values, numberOfSignificantValueDigits, keyed, formatter, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java index 7d6f65d418a6e..e169c3021c87c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java @@ -28,12 +28,11 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -42,13 +41,13 @@ public class StatsAggregationBuilder extends ValuesSourceAggregationBuilder.Leaf public static final String NAME = "stats"; public static final ObjectParser PARSER = - ObjectParser.fromBuilder(NAME, StatsAggregationBuilder::new); + ObjectParser.fromBuilder(NAME, StatsAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); } public StatsAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); } protected StatsAggregationBuilder(StatsAggregationBuilder clone, @@ -56,6 +55,10 @@ protected StatsAggregationBuilder(StatsAggregationBuilder clone, super(clone, factoriesBuilder, metaData); } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + StatsAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new StatsAggregationBuilder(this, factoriesBuilder, metaData); @@ -65,7 +68,12 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map config, + protected StatsAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new StatsAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java index 7ae5b016f75c3..bdb53e92df9eb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java @@ -49,7 +49,6 @@ class StatsAggregator extends NumericMetricsAggregator.MultiValue { DoubleArray mins; DoubleArray maxes; - StatsAggregator(String name, ValuesSource.Numeric valuesSource, DocValueFormat format, SearchContext context, Aggregator parent, List pipelineAggregators, Map metaData) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java index 6f71444007203..d1da48f0f5e9c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java @@ -20,24 +20,29 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class StatsAggregatorFactory extends ValuesSourceAggregatorFactory { +class StatsAggregatorFactory extends ValuesSourceAggregatorFactory { StatsAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, @@ -45,22 +50,45 @@ class StatsAggregatorFactory extends ValuesSourceAggregatorFactory pipelineAggregators, Map metaData) throws IOException { + return new StatsAggregator(name, (Numeric) valuesSource, formatter, context, parent, pipelineAggregators, metaData); + } + }); + } + @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, List pipelineAggregators, - Map metaData) - throws IOException { + Map metaData) throws IOException { return new StatsAggregator(name, null, config.format(), searchContext, parent, pipelineAggregators, metaData); } @Override - protected Aggregator doCreateInternal(Numeric valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new StatsAggregator(name, valuesSource, config.format(), searchContext, parent, pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + StatsAggregationBuilder.NAME); + + if (aggregatorSupplier instanceof MetricAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected MetricAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((MetricAggregatorSupplier) aggregatorSupplier).build(name, valuesSource, config.format(), searchContext, parent, + pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java index df6b1855f1f89..02c44597c54a6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java @@ -28,12 +28,11 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -43,11 +42,15 @@ public class SumAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, SumAggregationBuilder::new); static { - ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); + } + + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + SumAggregatorFactory.registerAggregators(valuesSourceRegistry); } public SumAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); } protected SumAggregationBuilder(SumAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -63,7 +66,12 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map config, + protected SumAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new SumAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java index 9e5473440425a..c581b2fcdc438 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java @@ -20,24 +20,29 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class SumAggregatorFactory extends ValuesSourceAggregatorFactory { +class SumAggregatorFactory extends ValuesSourceAggregatorFactory { SumAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, @@ -45,6 +50,22 @@ class SumAggregatorFactory extends ValuesSourceAggregatorFactory pipelineAggregators, Map metaData) throws IOException { + return new SumAggregator(name, (Numeric) valuesSource, formatter, context, parent, pipelineAggregators, metaData); + } + }); + } + @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, @@ -55,12 +76,20 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(Numeric valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new SumAggregator(name, valuesSource, config.format(), searchContext, parent, pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + SumAggregationBuilder.NAME); + + if (aggregatorSupplier instanceof MetricAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected MetricAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((MetricAggregatorSupplier) aggregatorSupplier).build(name, valuesSource, config.format(), searchContext, parent, + pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java index 04d83dae4e1ac..fdcd7e452fceb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java @@ -29,11 +29,11 @@ import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -42,13 +42,17 @@ public class ValueCountAggregationBuilder extends ValuesSourceAggregationBuilder public static final String NAME = "value_count"; public static final ObjectParser PARSER = - ObjectParser.fromBuilder(NAME, name -> new ValueCountAggregationBuilder(name, null)); + ObjectParser.fromBuilder(NAME, ValueCountAggregationBuilder::new); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); } - public ValueCountAggregationBuilder(String name, ValueType targetValueType) { - super(name, CoreValuesSourceType.ANY, targetValueType); + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + ValueCountAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + + public ValueCountAggregationBuilder(String name) { + super(name); } protected ValueCountAggregationBuilder(ValueCountAggregationBuilder clone, @@ -56,6 +60,11 @@ protected ValueCountAggregationBuilder(ValueCountAggregationBuilder clone, super(clone, factoriesBuilder, metaData); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metaData) { return new ValueCountAggregationBuilder(this, factoriesBuilder, metaData); @@ -65,7 +74,7 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu * Read from a stream. */ public ValueCountAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.ANY); + super(in); } @Override @@ -80,7 +89,7 @@ protected boolean serializeTargetValueType(Version version) { @Override protected ValueCountAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, + ValuesSourceConfig config, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException { return new ValueCountAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java index b00eaf6f27839..4f4d423944025 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java @@ -20,23 +20,42 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class ValueCountAggregatorFactory extends ValuesSourceAggregatorFactory { +class ValueCountAggregatorFactory extends ValuesSourceAggregatorFactory { - ValueCountAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, - AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.registerAny(ValueCountAggregationBuilder.NAME, + new ValueCountAggregatorSupplier() { + @Override + public Aggregator build(String name, + ValuesSource valuesSource, + SearchContext aggregationContext, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException { + return new ValueCountAggregator(name, valuesSource, aggregationContext, parent, pipelineAggregators, metaData); + } + }); + } + + ValueCountAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, + AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } @@ -55,6 +74,13 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new ValueCountAggregator(name, valuesSource, searchContext, parent, pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + ValueCountAggregationBuilder.NAME); + if (aggregatorSupplier instanceof ValueCountAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected ValueCountAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((ValueCountAggregatorSupplier) aggregatorSupplier) + .build(name, valuesSource, searchContext, parent, pipelineAggregators,metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorSupplier.java new file mode 100644 index 0000000000000..bbc47e6d56e84 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorSupplier.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface ValueCountAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + ValuesSource valuesSource, + SearchContext aggregationContext, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java index 48aa9fd1f922c..2d8548c68909e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java @@ -30,19 +30,20 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig; import org.elasticsearch.search.aggregations.support.MultiValuesSourceParseHelper; import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; import java.util.Objects; -public class WeightedAvgAggregationBuilder extends MultiValuesSourceAggregationBuilder.LeafOnly { +public class WeightedAvgAggregationBuilder extends MultiValuesSourceAggregationBuilder.LeafOnly { public static final String NAME = "weighted_avg"; public static final ParseField VALUE_FIELD = new ParseField("value"); public static final ParseField WEIGHT_FIELD = new ParseField("weight"); @@ -56,7 +57,7 @@ public class WeightedAvgAggregationBuilder extends MultiValuesSourceAggregationB } public WeightedAvgAggregationBuilder(String name) { - super(name, ValueType.NUMERIC); + super(name); } public WeightedAvgAggregationBuilder(WeightedAvgAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -79,7 +80,7 @@ public WeightedAvgAggregationBuilder weight(MultiValuesSourceFieldConfig weightC * Read from a stream. */ public WeightedAvgAggregationBuilder(StreamInput in) throws IOException { - super(in, ValueType.NUMERIC); + super(in); } @Override @@ -87,17 +88,22 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - Map> configs, - DocValueFormat format, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected MultiValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + Map configs, + DocValueFormat format, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return new WeightedAvgAggregatorFactory(name, configs, format, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java index 8bc8e4f85c435..a9fedbf0e7d3e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java @@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.MultiValuesSource; import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.internal.SearchContext; @@ -35,9 +34,9 @@ import java.util.List; import java.util.Map; -class WeightedAvgAggregatorFactory extends MultiValuesSourceAggregatorFactory { +class WeightedAvgAggregatorFactory extends MultiValuesSourceAggregatorFactory { - WeightedAvgAggregatorFactory(String name, Map> configs, + WeightedAvgAggregatorFactory(String name, Map configs, DocValueFormat format, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { @@ -54,7 +53,7 @@ protected Aggregator createUnmapped(SearchContext searchContext, @Override protected Aggregator doCreateInternal(SearchContext searchContext, - Map> configs, + Map configs, DocValueFormat format, Aggregator parent, boolean collectsFromSingleBucket, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregatorSupplier.java new file mode 100644 index 0000000000000..97ca793faedb3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregatorSupplier.java @@ -0,0 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.support; + +public interface AggregatorSupplier { +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index 6df32b4deefa0..e6064cbd1f35d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -21,54 +21,28 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.script.AggregationScript; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationExecutionException; -import java.io.IOException; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.Locale; import java.util.function.LongSupplier; /** * {@link CoreValuesSourceType} holds the {@link ValuesSourceType} implementations for the core aggregations package. */ -public enum CoreValuesSourceType implements Writeable, ValuesSourceType { - ANY { - // ANY still has a lot of special handling in ValuesSourceConfig, and as such doesn't adhere to this interface yet - @Override - public ValuesSource getEmpty() { - // TODO: Implement this or get rid of ANY - throw new UnsupportedOperationException("CoreValuesSourceType.ANY is still a special case"); - } - - @Override - public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { - // TODO: Implement this or get rid of ANY - throw new UnsupportedOperationException("CoreValuesSourceType.ANY is still a special case"); - } - - @Override - public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { - // TODO: Implement this or get rid of ANY - throw new UnsupportedOperationException("CoreValuesSourceType.ANY is still a special case"); - } - - @Override - public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { - return BYTES.replaceMissing(valuesSource, rawMissing, docValueFormat, now); - } - }, - NUMERIC { +public enum CoreValuesSourceType implements ValuesSourceType { + NUMERIC() { @Override public ValuesSource getEmpty() { return ValuesSource.Numeric.EMPTY; @@ -88,7 +62,7 @@ public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFa "], but got [" + fieldContext.fieldType().typeName() + "]"); } - ValuesSource.Numeric dataSource = new ValuesSource.Numeric.FieldData((IndexNumericFieldData)fieldContext.indexFieldData()); + ValuesSource.Numeric dataSource = new ValuesSource.Numeric.FieldData((IndexNumericFieldData) fieldContext.indexFieldData()); if (script != null) { // Value script case dataSource = new ValuesSource.Numeric.WithScript(dataSource, script); @@ -102,7 +76,7 @@ public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, return MissingValues.replaceMissing((ValuesSource.Numeric) valuesSource, missing); } }, - BYTES { + BYTES() { @Override public ValuesSource getEmpty() { return ValuesSource.Bytes.WithOrdinals.EMPTY; @@ -139,7 +113,7 @@ public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, } } }, - GEOPOINT { + GEOPOINT() { @Override public ValuesSource getEmpty() { return ValuesSource.GeoPoint.EMPTY; @@ -167,8 +141,13 @@ public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, final GeoPoint missing = new GeoPoint(rawMissing.toString()); return MissingValues.replaceMissing((ValuesSource.GeoPoint) valuesSource, missing); } + + @Override + public DocValueFormat getFormatter(String format, ZoneId tz) { + return DocValueFormat.GEOHASH; + } }, - RANGE { + RANGE() { @Override public ValuesSource getEmpty() { // TODO: Is this the correct exception type here? @@ -188,7 +167,7 @@ public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFa // TODO: Is this the correct exception type here? throw new IllegalStateException("Asked for range ValuesSource, but field is of type " + fieldType.name()); } - RangeFieldMapper.RangeFieldType rangeFieldType = (RangeFieldMapper.RangeFieldType)fieldType; + RangeFieldMapper.RangeFieldType rangeFieldType = (RangeFieldMapper.RangeFieldType) fieldType; return new ValuesSource.Range(fieldContext.indexFieldData(), rangeFieldType.rangeType()); } @@ -197,47 +176,93 @@ public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, throw new IllegalArgumentException("Can't apply missing values on a " + valuesSource.getClass()); } }, - HISTOGRAM { + IP() { @Override public ValuesSource getEmpty() { - // TODO: Is this the correct exception type here? - throw new IllegalArgumentException("Can't deal with unmapped ValuesSource type " + this.value()); + return BYTES.getEmpty(); } @Override public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { - throw new AggregationExecutionException("value source of type [" + this.value() + "] is not supported by scripts"); + return BYTES.getScript(script, scriptValueType); } @Override public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { - final IndexFieldData indexFieldData = fieldContext.indexFieldData(); + return BYTES.getField(fieldContext, script); + } - if (!(indexFieldData instanceof IndexHistogramFieldData)) { - throw new IllegalArgumentException("Expected histogram type on field [" + fieldContext.field() + - "], but got [" + fieldContext.fieldType().typeName() + "]"); - } - return new ValuesSource.Histogram.Fielddata((IndexHistogramFieldData) indexFieldData); + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + return BYTES.replaceMissing(valuesSource, rawMissing, docValueFormat, now); + } + + @Override + public DocValueFormat getFormatter(String format, ZoneId tz) { + return DocValueFormat.IP; + } + }, + DATE() { + @Override + public ValuesSource getEmpty() { + return NUMERIC.getEmpty(); + } + + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + return NUMERIC.getScript(script, scriptValueType); + } + + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + return NUMERIC.getField(fieldContext, script); } @Override public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { - throw new IllegalArgumentException("Can't apply missing values on a " + valuesSource.getClass()); + return NUMERIC.replaceMissing(valuesSource, rawMissing, docValueFormat, now); } - }; - public static ValuesSourceType fromString(String name) { - return valueOf(name.trim().toUpperCase(Locale.ROOT)); - } + @Override + public DocValueFormat getFormatter(String format, ZoneId tz) { + return new DocValueFormat.DateTime( + format == null ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER : DateFormatter.forPattern(format), + tz == null ? ZoneOffset.UTC : tz, + // If we were just looking at fields, we could read the resolution from the field settings, but we need to deal with script + // output, which has no way to indicate the resolution, so we need to default to something. Milliseconds is the standard. + DateFieldMapper.Resolution.MILLISECONDS); + } + }, + BOOLEAN() { + @Override + public ValuesSource getEmpty() { + return NUMERIC.getEmpty(); + } + + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + return NUMERIC.getScript(script, scriptValueType); + } + + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + return NUMERIC.getField(fieldContext, script); + } - public static ValuesSourceType fromStream(StreamInput in) throws IOException { - return in.readEnum(CoreValuesSourceType.class); + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + return NUMERIC.replaceMissing(valuesSource, rawMissing, docValueFormat, now); + } + + @Override + public DocValueFormat getFormatter(String format, ZoneId tz) { + return DocValueFormat.BOOLEAN; + } } + ; - @Override - public void writeTo(StreamOutput out) throws IOException { - CoreValuesSourceType state = this; - out.writeEnum(state); + public static ValuesSourceType fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); } public String value() { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/HistogramAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/HistogramAggregatorSupplier.java new file mode 100644 index 0000000000000..021577852c39e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/HistogramAggregatorSupplier.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.support; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface HistogramAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, AggregatorFactories factories, double interval, double offset, + BucketOrder order, boolean keyed, long minDocCount, double minBound, double maxBound, + @Nullable ValuesSource valuesSource, DocValueFormat formatter, + SearchContext context, Aggregator parent, + List pipelineAggregators, Map metaData) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java index 3a8bd9e12feca..fe1d5be993209 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import java.io.IOException; import java.util.HashMap; @@ -34,11 +35,16 @@ public abstract class MultiValuesSource { protected Map values; public static class NumericMultiValuesSource extends MultiValuesSource { - public NumericMultiValuesSource(Map> valuesSourceConfigs, + public NumericMultiValuesSource(Map valuesSourceConfigs, QueryShardContext context) { values = new HashMap<>(valuesSourceConfigs.size()); - for (Map.Entry> entry : valuesSourceConfigs.entrySet()) { - values.put(entry.getKey(), entry.getValue().toValuesSource(context)); + for (Map.Entry entry : valuesSourceConfigs.entrySet()) { + final ValuesSource valuesSource = entry.getValue().toValuesSource(); + if (valuesSource instanceof ValuesSource.Numeric == false) { + throw new AggregationExecutionException("ValuesSource type " + valuesSource.toString() + + "is not supported for multi-valued aggregation"); + } + values.put(entry.getKey(), (ValuesSource.Numeric) valuesSource); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java index efeed0c9efb39..1c36db7296472 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java @@ -40,18 +40,18 @@ * * A limitation of this class is that all the ValuesSource's being refereenced must be of the same type. */ -public abstract class MultiValuesSourceAggregationBuilder> +public abstract class MultiValuesSourceAggregationBuilder> extends AbstractAggregationBuilder { - public abstract static class LeafOnly> - extends MultiValuesSourceAggregationBuilder { + public abstract static class LeafOnly> + extends MultiValuesSourceAggregationBuilder { - protected LeafOnly(String name, ValueType targetValueType) { - super(name, targetValueType); + protected LeafOnly(String name) { + super(name); } - protected LeafOnly(LeafOnly clone, Builder factoriesBuilder, Map metaData) { + protected LeafOnly(LeafOnly clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); if (factoriesBuilder.count() > 0) { throw new AggregationInitializationException("Aggregator [" + name + "] of type [" @@ -62,8 +62,8 @@ protected LeafOnly(LeafOnly clone, Builder factoriesBuilder, Map fields = new HashMap<>(); - private final ValueType targetValueType; - private ValueType valueType = null; + private ValueType userValueTypeHint = null; private String format = null; - protected MultiValuesSourceAggregationBuilder(String name, ValueType targetValueType) { + protected MultiValuesSourceAggregationBuilder(String name) { super(name); - this.targetValueType = targetValueType; } - protected MultiValuesSourceAggregationBuilder(MultiValuesSourceAggregationBuilder clone, + protected MultiValuesSourceAggregationBuilder(MultiValuesSourceAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); this.fields = new HashMap<>(clone.fields); - this.targetValueType = clone.targetValueType; - this.valueType = clone.valueType; + this.userValueTypeHint = clone.userValueTypeHint; this.format = clone.format; } - protected MultiValuesSourceAggregationBuilder(StreamInput in, ValueType targetValueType) + /** + * Read from a stream. + */ + protected MultiValuesSourceAggregationBuilder(StreamInput in) throws IOException { super(in); - assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType"; - this.targetValueType = targetValueType; read(in); } @@ -109,17 +107,14 @@ protected MultiValuesSourceAggregationBuilder(StreamInput in, ValueType targetVa @SuppressWarnings("unchecked") private void read(StreamInput in) throws IOException { fields = in.readMap(StreamInput::readString, MultiValuesSourceFieldConfig::new); - valueType = in.readOptionalWriteable(ValueType::readFromStream); + userValueTypeHint = in.readOptionalWriteable(ValueType::readFromStream); format = in.readOptionalString(); } @Override protected final void doWriteTo(StreamOutput out) throws IOException { - if (serializeTargetValueType()) { - out.writeOptionalWriteable(targetValueType); - } out.writeMap(fields, StreamOutput::writeString, (o, value) -> value.writeTo(o)); - out.writeOptionalWriteable(valueType); + out.writeOptionalWriteable(userValueTypeHint); out.writeOptionalString(format); innerWriteTo(out); } @@ -142,11 +137,11 @@ protected AB field(String propertyName, MultiValuesSourceFieldConfig config) { * Sets the {@link ValueType} for the value produced by this aggregation */ @SuppressWarnings("unchecked") - public AB valueType(ValueType valueType) { + public AB userValueTypeHint(ValueType valueType) { if (valueType == null) { - throw new IllegalArgumentException("[valueType] must not be null: [" + name + "]"); + throw new IllegalArgumentException("[userValueTypeHint] must not be null: [" + name + "]"); } - this.valueType = valueType; + this.userValueTypeHint = valueType; return (AB) this; } @@ -162,25 +157,33 @@ public AB format(String format) { return (AB) this; } - @Override - protected final MultiValuesSourceAggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { - ValueType finalValueType = this.valueType != null ? this.valueType : targetValueType; + /** + * Aggregations should use this method to define a {@link ValuesSourceType} of last resort. This will only be used when the resolver + * can't find a field and the user hasn't provided a value type hint. + * + * @return The CoreValuesSourceType we expect this script to yield. + */ + protected abstract ValuesSourceType defaultValueSourceType(); - Map> configs = new HashMap<>(fields.size()); + @Override + protected final MultiValuesSourceAggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { + Map configs = new HashMap<>(fields.size()); fields.forEach((key, value) -> { - ValuesSourceConfig config = ValuesSourceConfig.resolve(queryShardContext, finalValueType, - value.getFieldName(), value.getScript(), value.getMissing(), value.getTimeZone(), format); + ValuesSourceConfig config = ValuesSourceConfig.resolveUnregistered(queryShardContext, userValueTypeHint, + value.getFieldName(), value.getScript(), value.getMissing(), value.getTimeZone(), format, defaultValueSourceType()); configs.put(key, config); }); - DocValueFormat docValueFormat = resolveFormat(format, finalValueType); + DocValueFormat docValueFormat = resolveFormat(format, userValueTypeHint, defaultValueSourceType()); return innerBuild(queryShardContext, configs, docValueFormat, parent, subFactoriesBuilder); } - private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValueType valueType) { + private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValueType valueType, + ValuesSourceType defaultValuesSourceType) { if (valueType == null) { - return DocValueFormat.RAW; // we can't figure it out + // If the user didn't send a hint, all we can do is fall back to the default + return defaultValuesSourceType.getFormatter(format, null); } DocValueFormat valueFormat = valueType.defaultFormat; if (valueFormat instanceof DocValueFormat.Decimal && format != null) { @@ -189,19 +192,11 @@ private static DocValueFormat resolveFormat(@Nullable String format, @Nullable V return valueFormat; } - protected abstract MultiValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - Map> configs, - DocValueFormat format, AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException; - + protected abstract MultiValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + Map configs, + DocValueFormat format, AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException; - /** - * Should this builder serialize its targetValueType? Defaults to false. All subclasses that override this to true - * should use the three argument read constructor rather than the four argument version. - */ - protected boolean serializeTargetValueType() { - return false; - } @Override public final XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException { @@ -214,8 +209,8 @@ public final XContentBuilder internalXContent(XContentBuilder builder, Params pa if (format != null) { builder.field(CommonFields.FORMAT.getPreferredName(), format); } - if (valueType != null) { - builder.field(CommonFields.VALUE_TYPE.getPreferredName(), valueType.getPreferredName()); + if (userValueTypeHint != null) { + builder.field(CommonFields.VALUE_TYPE.getPreferredName(), userValueTypeHint.getPreferredName()); } doXContentBody(builder, params); builder.endObject(); @@ -226,7 +221,7 @@ public final XContentBuilder internalXContent(XContentBuilder builder, Params pa @Override public int hashCode() { - return Objects.hash(super.hashCode(), fields, format, targetValueType, valueType); + return Objects.hash(super.hashCode(), fields, format, userValueTypeHint); } @@ -239,6 +234,6 @@ public boolean equals(Object obj) { MultiValuesSourceAggregationBuilder other = (MultiValuesSourceAggregationBuilder) obj; return Objects.equals(this.fields, other.fields) && Objects.equals(this.format, other.format) - && Objects.equals(this.valueType, other.valueType); + && Objects.equals(this.userValueTypeHint, other.userValueTypeHint); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java index a76f345071d35..711c53f13d9df 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java @@ -31,13 +31,12 @@ import java.util.List; import java.util.Map; -public abstract class MultiValuesSourceAggregatorFactory - extends AggregatorFactory { +public abstract class MultiValuesSourceAggregatorFactory extends AggregatorFactory { - protected final Map> configs; + protected final Map configs; protected final DocValueFormat format; - public MultiValuesSourceAggregatorFactory(String name, Map> configs, + public MultiValuesSourceAggregatorFactory(String name, Map configs, DocValueFormat format, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { @@ -62,7 +61,7 @@ protected abstract Aggregator createUnmapped(SearchContext searchContext, List pipelineAggregators, Map metaData) throws IOException; - protected abstract Aggregator doCreateInternal(SearchContext searchContext, Map> configs, + protected abstract Aggregator doCreateInternal(SearchContext searchContext, Map configs, DocValueFormat format, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java index 4888495f9d8da..bafa6426480be 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java @@ -27,17 +27,17 @@ public final class MultiValuesSourceParseHelper { - public static void declareCommon( - AbstractObjectParser, T> objectParser, boolean formattable, - ValueType targetValueType) { + public static void declareCommon( + AbstractObjectParser, T> objectParser, boolean formattable, + ValueType expectedValueType) { - objectParser.declareField(MultiValuesSourceAggregationBuilder::valueType, p -> { - ValueType valueType = ValueType.resolveForScript(p.text()); - if (targetValueType != null && valueType.isNotA(targetValueType)) { + objectParser.declareField(MultiValuesSourceAggregationBuilder::userValueTypeHint, p -> { + ValueType valueType = ValueType.lenientParse(p.text()); + if (expectedValueType != null && valueType.isNotA(expectedValueType)) { throw new ParsingException(p.getTokenLocation(), "Aggregation [" + objectParser.getName() + "] was configured with an incompatible value type [" - + valueType + "]. It can only work on value of type [" - + targetValueType + "]"); + + valueType + "]. It can only work on value off type [" + + expectedValueType + "]"); } return valueType; }, ValueType.VALUE_TYPE, ObjectParser.ValueType.STRING); @@ -49,7 +49,7 @@ public static void declareCommon( } public static void declareField(String fieldName, - AbstractObjectParser, T> objectParser, + AbstractObjectParser, T> objectParser, boolean scriptable, boolean timezoneAware) { objectParser.declareField((o, fieldConfig) -> o.field(fieldName, fieldConfig.build()), diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java index a1e3237288fa9..068790030bd49 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java @@ -23,38 +23,33 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.plain.BinaryDVIndexFieldData; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.search.DocValueFormat; import java.io.IOException; import java.time.ZoneOffset; +import java.util.Set; public enum ValueType implements Writeable { STRING((byte) 1, "string", "string", CoreValuesSourceType.BYTES, - IndexFieldData.class, DocValueFormat.RAW), - LONG((byte) 2, "byte|short|integer|long", "long", - CoreValuesSourceType.NUMERIC, - IndexNumericFieldData.class, DocValueFormat.RAW), - DOUBLE((byte) 3, "float|double", "double", CoreValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.RAW), - NUMBER((byte) 4, "number", "number", CoreValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.RAW), - DATE((byte) 5, "date", "date", CoreValuesSourceType.NUMERIC, IndexNumericFieldData.class, - new DocValueFormat.DateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ZoneOffset.UTC, + DocValueFormat.RAW), + + LONG((byte) 2, "byte|short|integer|long", "long", CoreValuesSourceType.NUMERIC, DocValueFormat.RAW), + DOUBLE((byte) 3, "float|double", "double", CoreValuesSourceType.NUMERIC, DocValueFormat.RAW), + NUMBER((byte) 4, "number", "number", CoreValuesSourceType.NUMERIC, DocValueFormat.RAW), + DATE((byte) 5, "date", "date", CoreValuesSourceType.DATE, + new DocValueFormat.DateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ZoneOffset.UTC, DateFieldMapper.Resolution.MILLISECONDS)), - IP((byte) 6, "ip", "ip", CoreValuesSourceType.BYTES, IndexFieldData.class, DocValueFormat.IP), + IP((byte) 6, "ip", "ip", CoreValuesSourceType.IP, DocValueFormat.IP), // TODO: what is the difference between "number" and "numeric"? - NUMERIC((byte) 7, "numeric", "numeric", CoreValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.RAW), - GEOPOINT((byte) 8, "geo_point", "geo_point", CoreValuesSourceType.GEOPOINT, IndexGeoPointFieldData.class, DocValueFormat.GEOHASH), - BOOLEAN((byte) 9, "boolean", "boolean", CoreValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.BOOLEAN), - RANGE((byte) 10, "range", "range", CoreValuesSourceType.RANGE, BinaryDVIndexFieldData.class, DocValueFormat.RAW); + NUMERIC((byte) 7, "numeric", "numeric", CoreValuesSourceType.NUMERIC, DocValueFormat.RAW), + GEOPOINT((byte) 8, "geo_point", "geo_point", CoreValuesSourceType.GEOPOINT, DocValueFormat.GEOHASH), + BOOLEAN((byte) 9, "boolean", "boolean", CoreValuesSourceType.BOOLEAN, DocValueFormat.BOOLEAN), + RANGE((byte) 10, "range", "range", CoreValuesSourceType.RANGE, DocValueFormat.RAW); final String description; final ValuesSourceType valuesSourceType; - final Class fieldDataType; final DocValueFormat defaultFormat; private final byte id; private String preferredName; @@ -62,12 +57,11 @@ public enum ValueType implements Writeable { public static final ParseField VALUE_TYPE = new ParseField("value_type", "valueType"); ValueType(byte id, String description, String preferredName, ValuesSourceType valuesSourceType, - Class fieldDataType, DocValueFormat defaultFormat) { + DocValueFormat defaultFormat) { this.id = id; this.description = description; this.preferredName = preferredName; this.valuesSourceType = valuesSourceType; - this.fieldDataType = fieldDataType; this.defaultFormat = defaultFormat; } @@ -79,9 +73,26 @@ public ValuesSourceType getValuesSourceType() { return valuesSourceType; } + private static Set numericValueTypes = Set.of(ValueType.DOUBLE, ValueType.DATE, ValueType.LONG, ValueType.NUMBER, + ValueType.NUMERIC, ValueType.BOOLEAN); + private static Set stringValueTypes = Set.of(ValueType.STRING, ValueType.IP); + + /** + * This is a bit of a hack to mirror the old {@link ValueType} behavior, which would allow a rough compatibility between types. This + * behavior is being phased out in the aggregations framework, in favor of explicitly listing supported types, but we haven't gotten + * to fixing composite yet. + * + * @param valueType The value type the user suggested + * @return True iff the two value types are interchangeable + */ public boolean isA(ValueType valueType) { - return valueType.valuesSourceType == valuesSourceType && - valueType.fieldDataType.isAssignableFrom(fieldDataType); + if (numericValueTypes.contains(this)) { + return numericValueTypes.contains(valueType); + } + if (stringValueTypes.contains(this)) { + return stringValueTypes.contains(valueType); + } + return this.equals(valueType); } public boolean isNotA(ValueType valueType) { @@ -92,7 +103,7 @@ public DocValueFormat defaultFormat() { return defaultFormat; } - public static ValueType resolveForScript(String type) { + public static ValueType lenientParse(String type) { switch (type) { case "string": return STRING; case "double": @@ -122,7 +133,7 @@ public static ValueType readFromStream(StreamInput in) throws IOException { return valueType; } } - throw new IOException("No valueType found for id [" + id + "]"); + throw new IOException("No ValueType found for id [" + id + "]"); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index 483c3847c38d7..4bd3acfeeb08f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -33,10 +33,8 @@ import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.fielddata.DocValueBits; -import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.fielddata.MultiGeoPointValues; @@ -565,39 +563,4 @@ public org.elasticsearch.index.fielddata.MultiGeoPointValues geoPointValues(Leaf } } } - - public abstract static class Histogram extends ValuesSource { - - public abstract HistogramValues getHistogramValues(LeafReaderContext context) throws IOException; - - public static class Fielddata extends Histogram { - - protected final IndexHistogramFieldData indexFieldData; - - public Fielddata(IndexHistogramFieldData indexFieldData) { - this.indexFieldData = indexFieldData; - } - - @Override - public SortedBinaryDocValues bytesValues(LeafReaderContext context) { - return indexFieldData.load(context).getBytesValues(); - } - - @Override - public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { - HistogramValues values = getHistogramValues(context); - return new DocValueBits() { - @Override - public boolean advanceExact(int doc) throws IOException { - return values.advanceExact(doc); - } - }; - } - - public HistogramValues getHistogramValues(LeafReaderContext context) throws IOException { - return indexFieldData.load(context).getHistogramValues(); - } - } - } - } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java index d5af003331ba8..f7bdecfa3e448 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java @@ -19,9 +19,13 @@ package org.elasticsearch.search.aggregations.support; import org.elasticsearch.Version; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.AbstractObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; @@ -31,17 +35,54 @@ import java.io.IOException; import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.Map; import java.util.Objects; -public abstract class ValuesSourceAggregationBuilder> +public abstract class ValuesSourceAggregationBuilder> extends AbstractAggregationBuilder { - public abstract static class LeafOnly> - extends ValuesSourceAggregationBuilder { + public static void declareFields( + AbstractObjectParser, T> objectParser, + boolean scriptable, boolean formattable, boolean timezoneAware) { - protected LeafOnly(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) { - super(name, valuesSourceType, targetValueType); + + objectParser.declareField(ValuesSourceAggregationBuilder::field, XContentParser::text, + ParseField.CommonFields.FIELD, ObjectParser.ValueType.STRING); + + objectParser.declareField(ValuesSourceAggregationBuilder::missing, XContentParser::objectText, + ParseField.CommonFields.MISSING, ObjectParser.ValueType.VALUE); + + objectParser.declareField(ValuesSourceAggregationBuilder::userValueTypeHint, p -> ValueType.lenientParse(p.text()), + ValueType.VALUE_TYPE, ObjectParser.ValueType.STRING); + + if (formattable) { + objectParser.declareField(ValuesSourceAggregationBuilder::format, XContentParser::text, + ParseField.CommonFields.FORMAT, ObjectParser.ValueType.STRING); + } + + if (scriptable) { + objectParser.declareField(ValuesSourceAggregationBuilder::script, + (parser, context) -> Script.parse(parser), + Script.SCRIPT_PARSE_FIELD, ObjectParser.ValueType.OBJECT_OR_STRING); + } + + if (timezoneAware) { + objectParser.declareField(ValuesSourceAggregationBuilder::timeZone, p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return ZoneId.of(p.text()); + } else { + return ZoneOffset.ofHours(p.intValue()); + } + }, ParseField.CommonFields.TIME_ZONE, ObjectParser.ValueType.LONG); + } + } + + public abstract static class LeafOnly> + extends ValuesSourceAggregationBuilder { + + protected LeafOnly(String name) { + super(name); } protected LeafOnly(LeafOnly clone, Builder factoriesBuilder, Map metaData) { @@ -53,18 +94,10 @@ protected LeafOnly(LeafOnly clone, Builder factoriesBuilder, Map config; + protected ValuesSourceConfig config; - protected ValuesSourceAggregationBuilder(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) { + protected ValuesSourceAggregationBuilder(String name) { super(name); - if (valuesSourceType == null) { - throw new IllegalArgumentException("[valuesSourceType] must not be null: [" + name + "]"); - } - this.valuesSourceType = valuesSourceType; - this.targetValueType = targetValueType; } - protected ValuesSourceAggregationBuilder(ValuesSourceAggregationBuilder clone, + protected ValuesSourceAggregationBuilder(ValuesSourceAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); - this.valuesSourceType = clone.valuesSourceType; - this.targetValueType = clone.targetValueType; this.field = clone.field; - this.valueType = clone.valueType; + this.userValueTypeHint = clone.userValueTypeHint; this.format = clone.format; this.missing = clone.missing; this.timeZone = clone.timeZone; @@ -108,36 +132,18 @@ protected ValuesSourceAggregationBuilder(ValuesSourceAggregationBuilder } /** - * Read an aggregation from a stream that has a sensible default for TargetValueType. This should be used by most subclasses. - * Subclasses needing to maintain backward compatibility to a version that did not serialize TargetValueType should use this - * constructor, providing the old, constant value for TargetValueType and override {@link #serializeTargetValueType(Version)} to return - * true only for versions that support the serialization. + * Read from a stream. */ - protected ValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType) + protected ValuesSourceAggregationBuilder(StreamInput in) throws IOException { super(in); - this.valuesSourceType = valuesSourceType; if (serializeTargetValueType(in.getVersion())) { - this.targetValueType = in.readOptionalWriteable(ValueType::readFromStream); - } else { - this.targetValueType = targetValueType; + ValueType valueType = in.readOptionalWriteable(ValueType::readFromStream); + assert valueType == null; } read(in); } - /** - * Read an aggregation from a stream that serializes its targetValueType. This should only be used by subclasses that override - * {@link #serializeTargetValueType(Version)} to return true. - */ - protected ValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType) throws IOException { - super(in); - // TODO: Can we get rid of this constructor and always use the three value version? Does this assert provide any value? - assert serializeTargetValueType(in.getVersion()) : "Wrong read constructor called for subclass that serializes its targetValueType"; - this.valuesSourceType = valuesSourceType; - this.targetValueType = in.readOptionalWriteable(ValueType::readFromStream); - read(in); - } - /** * Read from a stream. */ @@ -147,7 +153,7 @@ private void read(StreamInput in) throws IOException { script = new Script(in); } if (in.readBoolean()) { - valueType = ValueType.readFromStream(in); + userValueTypeHint = ValueType.readFromStream(in); } format = in.readOptionalString(); missing = in.readGenericValue(); @@ -157,7 +163,8 @@ private void read(StreamInput in) throws IOException { @Override protected final void doWriteTo(StreamOutput out) throws IOException { if (serializeTargetValueType(out.getVersion())) { - out.writeOptionalWriteable(targetValueType); + // TODO: deprecate this so we don't need to carry around a useless null in the wire format + out.writeOptionalWriteable(null); } out.writeOptionalString(field); boolean hasScript = script != null; @@ -165,10 +172,10 @@ protected final void doWriteTo(StreamOutput out) throws IOException { if (hasScript) { script.writeTo(out); } - boolean hasValueType = valueType != null; + boolean hasValueType = userValueTypeHint != null; out.writeBoolean(hasValueType); if (hasValueType) { - valueType.writeTo(out); + userValueTypeHint.writeTo(out); } out.writeOptionalString(format); out.writeGenericValue(missing); @@ -182,8 +189,10 @@ protected final void doWriteTo(StreamOutput out) throws IOException { protected abstract void innerWriteTo(StreamOutput out) throws IOException; /** - * Should this builder serialize its targetValueType? Defaults to false. All subclasses that override this to true should use the three - * argument read constructor rather than the four argument version. + * DO NOT OVERRIDE THIS! + * + * This method only exists for legacy support. No new aggregations need this, nor should they override it. + * * @param version For backwards compatibility, subclasses can change behavior based on the version */ protected boolean serializeTargetValueType(Version version) { @@ -229,22 +238,25 @@ public Script script() { } /** - * Sets the {@link ValueType} for the value produced by this aggregation + * This setter should only be used during parsing, to set the userValueTypeHint. This is information the user provides in the json + * query to indicate the output type of a script or the type of the 'missing' replacement value. + * @param valueType - The parsed {@link ValueType} based on the string the user specified + * @return - The modified builder instance, for chaining. */ @SuppressWarnings("unchecked") - public AB valueType(ValueType valueType) { + public AB userValueTypeHint(ValueType valueType) { if (valueType == null) { - throw new IllegalArgumentException("[valueType] must not be null: [" + name + "]"); + // TODO: This is nonsense. We allow the value to be null (via constructor), but don't allow it to be set to null. This means + // thing looking to copy settings (like RollupRequestTranslator) need to check if userValueTypeHint is not null, and then + // set it if and only if it is non-null. + throw new IllegalArgumentException("[userValueTypeHint] must not be null: [" + name + "]"); } - this.valueType = valueType; + this.userValueTypeHint = valueType; return (AB) this; } - /** - * Gets the {@link ValueType} for the value produced by this aggregation - */ - public ValueType valueType() { - return valueType; + public ValueType userValueTypeHint() { + return userValueTypeHint; } /** @@ -307,44 +319,30 @@ public ZoneId timeZone() { } @Override - protected final ValuesSourceAggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { - ValuesSourceConfig config = resolveConfig(queryShardContext); - ValuesSourceAggregatorFactory factory = innerBuild(queryShardContext, config, parent, subFactoriesBuilder); + protected final ValuesSourceAggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { + ValuesSourceConfig config = resolveConfig(queryShardContext); + ValuesSourceAggregatorFactory factory = innerBuild(queryShardContext, config, parent, subFactoriesBuilder); return factory; } /** - * Provide a hook for aggregations to have finer grained control of the CoreValuesSourceType for script values. This will only be - * called if the user did not supply a type hint for the script. The script object is provided for reference. + * Aggregations should use this method to define a {@link ValuesSourceType} of last resort. This will only be used when the resolver + * can't find a field and the user hasn't provided a value type hint. * - * @param script - The user supplied script * @return The CoreValuesSourceType we expect this script to yield. */ - protected ValuesSourceType resolveScriptAny(Script script) { - return CoreValuesSourceType.BYTES; - } - - /** - * Provide a hook for aggregations to have finer grained control of the ValueType for script values. This will only be called if the - * user did not supply a type hint for the script. The script object is provided for reference - * @param script - the user supplied script - * @return The ValueType we expect this script to yield - */ - protected ValueType defaultValueType(Script script) { - return valueType; - } + protected abstract ValuesSourceType defaultValueSourceType(); - protected ValuesSourceConfig resolveConfig(QueryShardContext queryShardContext) { - ValueType valueType = this.valueType != null ? this.valueType : targetValueType; + protected ValuesSourceConfig resolveConfig(QueryShardContext queryShardContext) { return ValuesSourceConfig.resolve(queryShardContext, - valueType, field, script, missing, timeZone, format, this::resolveScriptAny); + this.userValueTypeHint, field, script, missing, timeZone, format, this.defaultValueSourceType(), this.getType()); } - protected abstract ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException; + protected abstract ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException; @Override public final XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException { @@ -364,8 +362,8 @@ public final XContentBuilder internalXContent(XContentBuilder builder, Params pa if (timeZone != null) { builder.field("time_zone", timeZone.toString()); } - if (valueType != null) { - builder.field("value_type", valueType.getPreferredName()); + if (userValueTypeHint != null) { + builder.field("value_type", userValueTypeHint.getPreferredName()); } doXContentBody(builder, params); builder.endObject(); @@ -376,8 +374,7 @@ public final XContentBuilder internalXContent(XContentBuilder builder, Params pa @Override public int hashCode() { - return Objects.hash(super.hashCode(), field, format, missing, script, - targetValueType, timeZone, valueType, valuesSourceType); + return Objects.hash(super.hashCode(), field, format, missing, script, timeZone, userValueTypeHint); } @Override @@ -385,14 +382,12 @@ public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; if (super.equals(obj) == false) return false; - ValuesSourceAggregationBuilder other = (ValuesSourceAggregationBuilder) obj; - return Objects.equals(valuesSourceType, other.valuesSourceType) - && Objects.equals(field, other.field) + ValuesSourceAggregationBuilder other = (ValuesSourceAggregationBuilder) obj; + return Objects.equals(field, other.field) && Objects.equals(format, other.format) && Objects.equals(missing, other.missing) && Objects.equals(script, other.script) - && Objects.equals(targetValueType, other.targetValueType) && Objects.equals(timeZone, other.timeZone) - && Objects.equals(valueType, other.valueType); + && Objects.equals(userValueTypeHint, other.userValueTypeHint); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java index ae98a534e4fb2..d9cfc453d6e98 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java @@ -30,12 +30,13 @@ import java.util.List; import java.util.Map; -public abstract class ValuesSourceAggregatorFactory extends AggregatorFactory { +public abstract class ValuesSourceAggregatorFactory extends AggregatorFactory { - protected ValuesSourceConfig config; + protected ValuesSourceConfig config; - public ValuesSourceAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, - AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { + public ValuesSourceAggregatorFactory(String name, ValuesSourceConfig config, QueryShardContext queryShardContext, + AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, queryShardContext, parent, subFactoriesBuilder, metaData); this.config = config; } @@ -43,34 +44,19 @@ public ValuesSourceAggregatorFactory(String name, ValuesSourceConfig config, @Override public Aggregator createInternal(SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - VS vs = config.toValuesSource(queryShardContext, this::resolveMissingAny); + ValuesSource vs = config.toValuesSource(); if (vs == null) { return createUnmapped(searchContext, parent, pipelineAggregators, metaData); } return doCreateInternal(vs, searchContext, parent, collectsFromSingleBucket, pipelineAggregators, metaData); } - /** - * This method provides a hook for aggregations that need finer grained control over the ValuesSource selected when the user supplies a - * missing value and there is no mapped field to infer the type from. This will only be called for aggregations that specify the - * CoreValuesSourceType.ANY in their constructors (On the builder class). The user supplied object is passed as a parameter, so its - * type * may be inspected as needed. - * - * Generally, only the type of the returned ValuesSource is used, so returning the EMPTY instance of the chosen type is recommended. - * - * @param missing The user supplied missing value - * @return A ValuesSource instance compatible with the supplied parameter - */ - protected ValuesSource resolveMissingAny(Object missing) { - return ValuesSource.Bytes.WithOrdinals.EMPTY; - } - protected abstract Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, List pipelineAggregators, Map metaData) throws IOException; - protected abstract Aggregator doCreateInternal(VS valuesSource, + protected abstract Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index ba599827b88af..6cd62a0d2a622 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -19,12 +19,9 @@ package org.elasticsearch.search.aggregations.support; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.query.QueryShardContext; @@ -33,105 +30,183 @@ import org.elasticsearch.search.DocValueFormat; import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.function.Function; +import java.util.function.LongSupplier; /** * A configuration that tells aggregations how to retrieve data from the index * in order to run a specific aggregation. */ -public class ValuesSourceConfig { +public class ValuesSourceConfig { /** - * Resolve a {@link ValuesSourceConfig} given configuration parameters. + * Given the query context and other information, decide on the input {@link ValuesSource} for this aggretation run, and construct a new + * {@link ValuesSourceConfig} based on that {@link ValuesSourceType} + * + * @param context - the query context + * @param userValueTypeHint - User specified value type; used for missing values and scripts + * @param field - The field being aggregated over. At least one of field and script must not be null + * @param script - The script the user specified. At least one of field and script must not be null + * @param missing - A user specified value to apply when the field is missing. Should be of type userValueTypeHint + * @param timeZone - Used to generate a format for dates + * @param format - The format string to apply to this field. Confusingly, this is used for input parsing as well as output formatting + * See https://github.com/elastic/elasticsearch/issues/47469 + * @param defaultValueSourceType - per-aggregation {@link ValuesSource} of last resort. + * @param aggregationName - Name of the aggregation, generally from the aggregation builder. This is used as a lookup key in the + * {@link ValuesSourceRegistry} + * @return - An initialized {@link ValuesSourceConfig} that will yield the appropriate {@link ValuesSourceType} */ - public static ValuesSourceConfig resolve( - QueryShardContext context, - ValueType valueType, - String field, Script script, - Object missing, - ZoneId timeZone, - String format) { - return resolve(context, valueType, field, script, missing, timeZone, format, s -> CoreValuesSourceType.BYTES); + public static ValuesSourceConfig resolve(QueryShardContext context, + ValueType userValueTypeHint, + String field, + Script script, + Object missing, + ZoneId timeZone, + String format, + ValuesSourceType defaultValueSourceType, + String aggregationName) { + + return internalResolve(context, userValueTypeHint, field, script, missing, timeZone, format, defaultValueSourceType, + aggregationName, ValuesSourceConfig::getMappingFromRegistry); } /** - * Resolve a {@link ValuesSourceConfig} given configuration parameters. + * AKA legacy resolve. This method should be called by aggregations not supported by the {@link ValuesSourceRegistry}, to use the + * pre-registry logic to decide on the {@link ValuesSourceType}. New aggregations which extend from + * {@link ValuesSourceAggregationBuilder} should not use this method, preferring {@link ValuesSourceConfig#resolve} instead. + * + * @param context - the query context + * @param userValueTypeHint - User specified value type; used for missing values and scripts + * @param field - The field being aggregated over. At least one of field and script must not be null + * @param script - The script the user specified. At least one of field and script must not be null + * @param missing - A user specified value to apply when the field is missing. Should be of type userValueTypeHint + * @param timeZone - Used to generate a format for dates + * @param format - The format string to apply to this field. Confusingly, this is used for input parsing as well as output formatting + * See https://github.com/elastic/elasticsearch/issues/47469 + * @param defaultValueSourceType - per-aggregation {@link ValuesSource} of last resort. + * @return - An initialized {@link ValuesSourceConfig} that will yield the appropriate {@link ValuesSourceType} */ - public static ValuesSourceConfig resolve( - QueryShardContext context, - ValueType valueType, - String field, Script script, - Object missing, - ZoneId timeZone, - String format, - Function resolveScriptAny - ) { + public static ValuesSourceConfig resolveUnregistered(QueryShardContext context, + ValueType userValueTypeHint, + String field, + Script script, + Object missing, + ZoneId timeZone, + String format, + ValuesSourceType defaultValueSourceType) { + return internalResolve(context, userValueTypeHint, field, script, missing, timeZone, format, defaultValueSourceType, null, + ValuesSourceConfig::getLegacyMapping); + } + private static ValuesSourceConfig internalResolve(QueryShardContext context, + ValueType userValueTypeHint, + String field, + Script script, + Object missing, + ZoneId timeZone, + String format, + ValuesSourceType defaultValueSourceType, + String aggregationName, + FieldResolver fieldResolver + ) { + ValuesSourceConfig config; + MappedFieldType fieldType = null; + ValuesSourceType valuesSourceType; + ValueType scriptValueType = null; + AggregationScript.LeafFactory aggregationScript = null; + boolean unmapped = false; if (field == null) { + // Stand Alone Script Case if (script == null) { - ValuesSourceConfig config = new ValuesSourceConfig<>(CoreValuesSourceType.ANY); - config.format(resolveFormat(null, valueType, timeZone)); - return config; + throw new IllegalStateException( + "value source config is invalid; must have either a field context or a script or marked as unmapped"); } - ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : CoreValuesSourceType.ANY; - if (valuesSourceType == CoreValuesSourceType.ANY) { - // the specific value source type is undefined, but for scripts, - // we need to have a specific value source - // type to know how to handle the script values, so we fallback - // on Bytes - valuesSourceType = resolveScriptAny.apply(script); + /* + * This is the Stand Alone Script path. We should have a script that will produce a value independent of the presence or + * absence of any one field. The type of the script is given by the userValueTypeHint field, if the user specified a type, + * or the aggregation's default type if the user didn't. + */ + if (userValueTypeHint != null) { + valuesSourceType = userValueTypeHint.getValuesSourceType(); + } else { + valuesSourceType = defaultValueSourceType; } - ValuesSourceConfig config = new ValuesSourceConfig<>(valuesSourceType); - config.missing(missing); - config.timezone(timeZone); - config.format(resolveFormat(format, valueType, timeZone)); - config.script(createScript(script, context)); - config.scriptValueType(valueType); - return config; - } - - MappedFieldType fieldType = context.fieldMapper(field); - if (fieldType == null) { - ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : CoreValuesSourceType.ANY; - ValuesSourceConfig config = new ValuesSourceConfig<>(valuesSourceType); - config.missing(missing); - config.timezone(timeZone); - config.format(resolveFormat(format, valueType, timeZone)); - config.unmapped(true); - if (valueType != null) { - // todo do we really need this for unmapped? - config.scriptValueType(valueType); + aggregationScript = createScript(script, context); + scriptValueType = userValueTypeHint; + } else { + // Field case + fieldType = context.fieldMapper(field); + if (fieldType == null) { + /* Unmapped Field Case + * We got here because the user specified a field, but it doesn't exist on this index, possibly because of a wildcard index + * pattern. In this case, we're going to end up using the EMPTY variant of the ValuesSource, and possibly applying a user + * specified missing value. + */ + if (userValueTypeHint != null) { + valuesSourceType = userValueTypeHint.getValuesSourceType(); + } else { + valuesSourceType = defaultValueSourceType; + } + unmapped = true; + if (userValueTypeHint != null) { + // todo do we really need this for unmapped? + scriptValueType = userValueTypeHint; + } + } else { + valuesSourceType = fieldResolver.getValuesSourceType(context, fieldType, aggregationName, userValueTypeHint, + defaultValueSourceType); + aggregationScript = createScript(script, context); } - return config; } + config = new ValuesSourceConfig(valuesSourceType, fieldType, unmapped, aggregationScript, scriptValueType , context); + config.format(resolveFormat(format, valuesSourceType, timeZone, fieldType)); + config.missing(missing); + config.timezone(timeZone); + return config; + } + + @FunctionalInterface + private interface FieldResolver { + ValuesSourceType getValuesSourceType( + QueryShardContext context, + MappedFieldType fieldType, + String aggregationName, + ValueType userValueTypeHint, + ValuesSourceType defaultValuesSourceType); + + } + private static ValuesSourceType getMappingFromRegistry( + QueryShardContext context, + MappedFieldType fieldType, + String aggregationName, + ValueType userValueTypeHint, + ValuesSourceType defaultValuesSourceType) { IndexFieldData indexFieldData = context.getForField(fieldType); + return context.getValuesSourceRegistry().getValuesSourceType(fieldType, aggregationName, indexFieldData, + userValueTypeHint, defaultValuesSourceType); + } - ValuesSourceConfig config; + private static ValuesSourceType getLegacyMapping( + QueryShardContext context, + MappedFieldType fieldType, + String aggregationName, + ValueType userValueTypeHint, + ValuesSourceType defaultValuesSourceType) { + IndexFieldData indexFieldData = context.getForField(fieldType); if (indexFieldData instanceof IndexNumericFieldData) { - config = new ValuesSourceConfig<>(CoreValuesSourceType.NUMERIC); + return CoreValuesSourceType.NUMERIC; } else if (indexFieldData instanceof IndexGeoPointFieldData) { - config = new ValuesSourceConfig<>(CoreValuesSourceType.GEOPOINT); + return CoreValuesSourceType.GEOPOINT; } else if (fieldType instanceof RangeFieldMapper.RangeFieldType) { - config = new ValuesSourceConfig<>(CoreValuesSourceType.RANGE); - } else if (indexFieldData instanceof IndexHistogramFieldData) { - config = new ValuesSourceConfig<>(CoreValuesSourceType.HISTOGRAM); + return CoreValuesSourceType.RANGE; } else { - if (valueType == null) { - config = new ValuesSourceConfig<>(CoreValuesSourceType.BYTES); + if (userValueTypeHint == null) { + return defaultValuesSourceType; } else { - config = new ValuesSourceConfig<>(valueType.getValuesSourceType()); + return userValueTypeHint.getValuesSourceType(); } } - - config.fieldContext(new FieldContext(field, indexFieldData, fieldType)); - config.missing(missing); - config.timezone(timeZone); - config.script(createScript(script, context)); - config.format(fieldType.docValueFormat(format, timeZone)); - return config; } private static AggregationScript.LeafFactory createScript(Script script, QueryShardContext context) { @@ -143,36 +218,65 @@ private static AggregationScript.LeafFactory createScript(Script script, QuerySh } } - private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValueType valueType, @Nullable ZoneId tz) { - if (valueType == null) { - return DocValueFormat.RAW; // we can't figure it out - } - DocValueFormat valueFormat = valueType.defaultFormat; - if (valueFormat instanceof DocValueFormat.Decimal && format != null) { - valueFormat = new DocValueFormat.Decimal(format); - } - if (valueFormat instanceof DocValueFormat.DateTime && format != null) { - valueFormat = new DocValueFormat.DateTime(DateFormatter.forPattern(format), tz != null ? tz : ZoneOffset.UTC, - DateFieldMapper.Resolution.MILLISECONDS); + private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValuesSourceType valuesSourceType, @Nullable ZoneId tz, + MappedFieldType fieldType) { + if (fieldType != null) { + return fieldType.docValueFormat(format, tz); } - return valueFormat; + // Script or Unmapped case + return valuesSourceType.getFormatter(format, tz); } - private final ValuesSourceType valueSourceType; + /** + * Special case factory method, intended to be used by aggregations which have some specialized logic for figuring out what field they + * are operating on, for example Parent and Child join aggregations, which use the join relation to find the field they are reading from + * rather than a user specified field. + */ + public static ValuesSourceConfig resolveFieldOnly(MappedFieldType fieldType, + QueryShardContext queryShardContext) { + return new ValuesSourceConfig(fieldType.getValuesSourceType(), fieldType, false, null, null, queryShardContext); + } + + /** + * Convenience method for creating unmapped configs + */ + public static ValuesSourceConfig resolveUnmapped(ValuesSourceType valuesSourceType, QueryShardContext queryShardContext) { + return new ValuesSourceConfig(valuesSourceType, null, true, null, null, queryShardContext); + } + + private final ValuesSourceType valuesSourceType; private FieldContext fieldContext; private AggregationScript.LeafFactory script; private ValueType scriptValueType; - private boolean unmapped = false; + private boolean unmapped; private DocValueFormat format = DocValueFormat.RAW; private Object missing; private ZoneId timeZone; + private LongSupplier nowSupplier; + + + public ValuesSourceConfig(ValuesSourceType valuesSourceType, + MappedFieldType fieldType, + boolean unmapped, + AggregationScript.LeafFactory script, + ValueType scriptValueType, + QueryShardContext queryShardContext) { + if (unmapped && fieldType != null) { + throw new IllegalStateException("value source config is invalid; marked as unmapped but specified a mapped field"); + } + this.valuesSourceType = valuesSourceType; + if (fieldType != null) { + this.fieldContext = new FieldContext(fieldType.name(), queryShardContext.getForField(fieldType), fieldType); + } + this.unmapped = unmapped; + this.script = script; + this.scriptValueType = scriptValueType; + this.nowSupplier = queryShardContext::nowInMillis; - public ValuesSourceConfig(ValuesSourceType valueSourceType) { - this.valueSourceType = valueSourceType; } public ValuesSourceType valueSourceType() { - return valueSourceType; + return valuesSourceType; } public FieldContext fieldContext() { @@ -191,36 +295,16 @@ public boolean valid() { return fieldContext != null || script != null || unmapped; } - public ValuesSourceConfig fieldContext(FieldContext fieldContext) { - this.fieldContext = fieldContext; - return this; - } - - public ValuesSourceConfig script(AggregationScript.LeafFactory script) { - this.script = script; - return this; - } - - public ValuesSourceConfig scriptValueType(ValueType scriptValueType) { - this.scriptValueType = scriptValueType; - return this; - } - public ValueType scriptValueType() { return this.scriptValueType; } - public ValuesSourceConfig unmapped(boolean unmapped) { - this.unmapped = unmapped; - return this; - } - - public ValuesSourceConfig format(final DocValueFormat format) { + private ValuesSourceConfig format(final DocValueFormat format) { this.format = format; return this; } - public ValuesSourceConfig missing(final Object missing) { + private ValuesSourceConfig missing(final Object missing) { this.missing = missing; return this; } @@ -229,7 +313,7 @@ public Object missing() { return this.missing; } - public ValuesSourceConfig timezone(final ZoneId timeZone) { + private ValuesSourceConfig timezone(final ZoneId timeZone) { this.timeZone = timeZone; return this; } @@ -242,49 +326,41 @@ public DocValueFormat format() { return format; } + /** + * Transform the {@link ValuesSourceType} we selected in resolve into the specific {@link ValuesSource} instance to use for this shard + * @return - A {@link ValuesSource} ready to be read from by an aggregator + */ @Nullable - public VS toValuesSource(QueryShardContext context) { - return toValuesSource(context, value -> ValuesSource.Bytes.WithOrdinals.EMPTY); - } - - /** Get a value source given its configuration. A return value of null indicates that - * no value source could be built. */ - @Nullable - public VS toValuesSource(QueryShardContext context, Function resolveMissingAny) { + public ValuesSource toValuesSource() { if (!valid()) { + // TODO: resolve no longer generates invalid configs. Once VSConfig is immutable, we can drop this check throw new IllegalStateException( "value source config is invalid; must have either a field context or a script or marked as unwrapped"); } - final VS vs; + final ValuesSource vs; if (unmapped()) { if (missing() == null) { - // otherwise we will have values because of the missing value + /* Null values source signals to the AggregationBuilder to use the createUnmapped method, which aggregator factories can + * override to provide an aggregator optimized to return empty values + */ vs = null; - } else if (valueSourceType() == CoreValuesSourceType.ANY) { - // TODO: Clean up special cases around CoreValuesSourceType.ANY - vs = (VS) resolveMissingAny.apply(missing()); } else { - vs = (VS) valueSourceType().getEmpty(); + vs = valueSourceType().getEmpty(); } } else { if (fieldContext() == null) { - vs = (VS) valueSourceType().getScript(script(), scriptValueType()); + // Script case + vs = valueSourceType().getScript(script(), scriptValueType()); } else { - if (valueSourceType() == CoreValuesSourceType.ANY) { - // TODO: Clean up special cases around CoreValuesSourceType.ANY - // falling back to bytes values - vs = (VS) CoreValuesSourceType.BYTES.getField(fieldContext(), script()); - } else { - // TODO: Better docs for Scripts vs Scripted Fields - vs = (VS) valueSourceType().getField(fieldContext(), script()); - } + // Field or Value Script case + vs = valueSourceType().getField(fieldContext(), script()); } } if (missing() == null) { return vs; } - return (VS) valueSourceType().replaceMissing(vs, missing, format, context::nowInMillis); + return valueSourceType().replaceMissing(vs, missing, format, nowSupplier); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java deleted file mode 100644 index 567862ca92e3e..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.support; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.xcontent.AbstractObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.script.Script; - -import java.time.ZoneId; -import java.time.ZoneOffset; - -public final class ValuesSourceParserHelper { - - private ValuesSourceParserHelper() {} // utility class, no instantiation - - public static void declareAnyFields( - AbstractObjectParser, T> objectParser, - boolean scriptable, boolean formattable) { - declareAnyFields(objectParser, scriptable, formattable, false); - } - - public static void declareAnyFields( - AbstractObjectParser, T> objectParser, - boolean scriptable, boolean formattable, boolean timezoneAware) { - declareFields(objectParser, scriptable, formattable, timezoneAware, null); - } - - public static void declareNumericFields( - AbstractObjectParser, T> objectParser, - boolean scriptable, boolean formattable, boolean timezoneAware) { - declareFields(objectParser, scriptable, formattable, timezoneAware, ValueType.NUMERIC); - } - - public static void declareBytesFields( - AbstractObjectParser, T> objectParser, - boolean scriptable, boolean formattable) { - declareFields(objectParser, scriptable, formattable, false, ValueType.STRING); - } - - public static void declareGeoFields( - AbstractObjectParser, T> objectParser, - boolean scriptable, boolean formattable) { - declareFields(objectParser, scriptable, formattable, false, ValueType.GEOPOINT); - } - - private static void declareFields( - AbstractObjectParser, T> objectParser, - boolean scriptable, boolean formattable, boolean timezoneAware, ValueType targetValueType) { - - - objectParser.declareField(ValuesSourceAggregationBuilder::field, XContentParser::text, - ParseField.CommonFields.FIELD, ObjectParser.ValueType.STRING); - - objectParser.declareField(ValuesSourceAggregationBuilder::missing, XContentParser::objectText, - ParseField.CommonFields.MISSING, ObjectParser.ValueType.VALUE); - - objectParser.declareField(ValuesSourceAggregationBuilder::valueType, p -> { - ValueType valueType = ValueType.resolveForScript(p.text()); - if (targetValueType != null && valueType.isNotA(targetValueType)) { - throw new ParsingException(p.getTokenLocation(), - "Aggregation [" + objectParser.getName() + "] was configured with an incompatible value type [" - + valueType + "]. It can only work on value of type [" - + targetValueType + "]"); - } - return valueType; - }, ValueType.VALUE_TYPE, ObjectParser.ValueType.STRING); - - if (formattable) { - objectParser.declareField(ValuesSourceAggregationBuilder::format, XContentParser::text, - ParseField.CommonFields.FORMAT, ObjectParser.ValueType.STRING); - } - - if (scriptable) { - objectParser.declareField(ValuesSourceAggregationBuilder::script, - (parser, context) -> Script.parse(parser), - Script.SCRIPT_PARSE_FIELD, ObjectParser.ValueType.OBJECT_OR_STRING); - } - - if (timezoneAware) { - objectParser.declareField(ValuesSourceAggregationBuilder::timeZone, p -> { - if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return ZoneId.of(p.text()); - } else { - return ZoneOffset.ofHours(p.intValue()); - } - }, ParseField.CommonFields.TIME_ZONE, ObjectParser.ValueType.LONG); - } - } - - - -} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java new file mode 100644 index 0000000000000..5356821c5f58f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java @@ -0,0 +1,193 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.support; + +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.AggregationExecutionException; + +import java.util.AbstractMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + +/** + * {@link ValuesSourceRegistry} holds the mapping from {@link ValuesSourceType}s to {@link AggregatorSupplier}s. DO NOT directly + * instantiate this class, instead get an already-configured copy from {@link QueryShardContext#getValuesSourceRegistry()}, or (in the case + * of some test scenarios only) directly from {@link SearchModule#getValuesSourceRegistry()} + * + */ +public class ValuesSourceRegistry { + // Maps Aggregation names to (ValuesSourceType, Supplier) pairs, keyed by ValuesSourceType + private Map, AggregatorSupplier>>> aggregatorRegistry = Map.of(); + + /** + * Register a ValuesSource to Aggregator mapping. + * + * Threading behavior notes: This call is both synchronized and expensive. It copies the entire existing mapping structure each + * time it is invoked. We expect that register will be called a small number of times during startup only (as plugins are being + * registered) and we can tolerate the cost at that time. Once all plugins are registered, we should never need to call register + * again. Comparatively, we expect to do many reads from the registry data structures, and those reads may be interleaved on + * different worker threads. Thus we want to optimize the read case to be thread safe and fast, which the immutable + * collections do well. Using immutable collections requires a copy on write mechanic, thus the somewhat non-intuitive + * implementation of this method. + * @param aggregationName The name of the family of aggregations, typically found via {@link ValuesSourceAggregationBuilder#getType()} + * @param appliesTo A predicate which accepts the resolved {@link ValuesSourceType} and decides if the given aggregator can be applied + * to that type. + * @param aggregatorSupplier An Aggregation-specific specialization of AggregatorSupplier which will construct the mapped aggregator + */ + public synchronized void register(String aggregationName, Predicate appliesTo, + AggregatorSupplier aggregatorSupplier) { + AbstractMap.SimpleEntry[] mappings; + if (aggregatorRegistry.containsKey(aggregationName)) { + List currentMappings = aggregatorRegistry.get(aggregationName); + mappings = (AbstractMap.SimpleEntry[]) currentMappings.toArray(new AbstractMap.SimpleEntry[currentMappings.size() + 1]); + } else { + mappings = new AbstractMap.SimpleEntry[1]; + } + mappings[mappings.length - 1] = new AbstractMap.SimpleEntry<>(appliesTo, aggregatorSupplier); + aggregatorRegistry = copyAndAdd(aggregatorRegistry,new AbstractMap.SimpleEntry<>(aggregationName, List.of(mappings))); + } + + /** + * Register a ValuesSource to Aggregator mapping. This version provides a convenience method for mappings that only apply to a single + * {@link ValuesSourceType}, to allow passing in the type and auto-wrapping it in a predicate + * @param aggregationName The name of the family of aggregations, typically found via {@link ValuesSourceAggregationBuilder#getType()} + * @param valuesSourceType The ValuesSourceType this mapping applies to. + * @param aggregatorSupplier An Aggregation-specific specialization of AggregatorSupplier which will construct the mapped aggregator + * from the aggregation standard set of parameters + */ + public void register(String aggregationName, ValuesSourceType valuesSourceType, AggregatorSupplier aggregatorSupplier) { + register(aggregationName, (candidate) -> valuesSourceType.equals(candidate), aggregatorSupplier); + } + + /** + * Register a ValuesSource to Aggregator mapping. This version provides a convenience method for mappings that only apply to a known + * list of {@link ValuesSourceType}, to allow passing in the type and auto-wrapping it in a predicate + * @param aggregationName The name of the family of aggregations, typically found via {@link ValuesSourceAggregationBuilder#getType()} + * @param valuesSourceTypes The ValuesSourceTypes this mapping applies to. + * @param aggregatorSupplier An Aggregation-specific specialization of AggregatorSupplier which will construct the mapped aggregator + * from the aggregation standard set of parameters + */ + public void register(String aggregationName, List valuesSourceTypes, AggregatorSupplier aggregatorSupplier) { + register(aggregationName, (candidate) -> { + for (ValuesSourceType valuesSourceType : valuesSourceTypes) { + if (valuesSourceType.equals(candidate)) { + return true; + } + } + return false; + }, aggregatorSupplier); + } + + /** + * Register an aggregator that applies to any values source type. This is a convenience method for aggregations that do not care at all + * about the types of their inputs. Aggregations using this version of registration should not make any other registrations, as the + * aggregator registered using this function will be applied in all cases. + * + * @param aggregationName The name of the family of aggregations, typically found via {@link ValuesSourceAggregationBuilder#getType()} + * @param aggregatorSupplier An Aggregation-specific specialization of AggregatorSupplier which will construct the mapped aggregator + * from the aggregation standard set of parameters. + */ + public void registerAny(String aggregationName, AggregatorSupplier aggregatorSupplier) { + register(aggregationName, (ignored) -> true, aggregatorSupplier); + } + + private AggregatorSupplier findMatchingSuppier(ValuesSourceType valuesSourceType, + List, AggregatorSupplier>> supportedTypes) { + for (Map.Entry, AggregatorSupplier> candidate : supportedTypes) { + if (candidate.getKey().test(valuesSourceType)) { + return candidate.getValue(); + } + } + return null; + } + + public AggregatorSupplier getAggregator(ValuesSourceType valuesSourceType, String aggregationName) { + if (aggregationName != null && aggregatorRegistry.containsKey(aggregationName)) { + AggregatorSupplier supplier = findMatchingSuppier(valuesSourceType, aggregatorRegistry.get(aggregationName)); + if (supplier == null) { + throw new AggregationExecutionException("ValuesSource type " + valuesSourceType.toString() + + " is not supported for aggregation" + aggregationName); + } + return supplier; + } + throw new AggregationExecutionException("Unregistered Aggregation [" + aggregationName + "]"); + } + + public ValuesSourceType getValuesSourceType(MappedFieldType fieldType, String aggregationName, + // TODO: the following arguments are only needed for the legacy case + IndexFieldData indexFieldData, + ValueType valueType, + ValuesSourceType defaultValuesSourceType) { + if (aggregationName != null && aggregatorRegistry.containsKey(aggregationName)) { + // This will throw if the field doesn't support values sources, although really we probably threw much earlier in that case + ValuesSourceType valuesSourceType = fieldType.getValuesSourceType(); + if (aggregatorRegistry.get(aggregationName) != null + && findMatchingSuppier(valuesSourceType, aggregatorRegistry.get(aggregationName)) != null) { + return valuesSourceType; + } + String fieldDescription = fieldType.typeName() + "(" + fieldType.toString() + ")"; + throw new IllegalArgumentException("Field [" + fieldType.name() + "] of type [" + fieldDescription + + "] is not supported for aggregation [" + aggregationName + "]"); + } else { + // TODO: Legacy resolve logic; remove this after converting all aggregations to the new system + if (indexFieldData instanceof IndexNumericFieldData) { + return CoreValuesSourceType.NUMERIC; + } else if (indexFieldData instanceof IndexGeoPointFieldData) { + return CoreValuesSourceType.GEOPOINT; + } else if (fieldType instanceof RangeFieldMapper.RangeFieldType) { + return CoreValuesSourceType.RANGE; + } else { + if (valueType == null) { + return defaultValuesSourceType; + } else { + return valueType.getValuesSourceType(); + } + } + } + } + + private static Map copyAndAdd(Map source, Map.Entry newValue) { + Map.Entry[] entries; + if (source.containsKey(newValue.getKey())) { + // Replace with new value + entries = new Map.Entry[source.size()]; + int i = 0; + for (Map.Entry entry : source.entrySet()) { + if (entry.getKey() == newValue.getKey()) { + entries[i] = newValue; + } else { + entries[i] = entry; + } + i++; + } + } else { + entries = source.entrySet().toArray(new Map.Entry[source.size() + 1]); + entries[entries.length - 1] = newValue; + } + return Map.ofEntries(entries); + } + +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java index 52f39cfe9326f..040ede186c7ab 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java @@ -22,6 +22,7 @@ import org.elasticsearch.script.AggregationScript; import org.elasticsearch.search.DocValueFormat; +import java.time.ZoneId; import java.util.function.LongSupplier; /** @@ -70,4 +71,18 @@ public interface ValuesSourceType { */ ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now); + + /** + * This method provides a hook for specifying a type-specific formatter. When {@link ValuesSourceConfig} can resolve a + * {@link org.elasticsearch.index.mapper.MappedFieldType}, it prefers to get the formatter from there. Only when a field can't be + * resolved (which is to say script cases and unmapped field cases), it will fall back to calling this method on whatever + * {@link ValuesSourceType} it was able to resolve to. + * + * @param format - User supplied format string (Optional) + * @param tz - User supplied time zone (Optional) + * @return - A formatter object, configured with the passed in settings if appropriate. + */ + default DocValueFormat getFormatter(String format, ZoneId tz) { + return DocValueFormat.RAW; + } } diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java index 10f252c30dc3b..7e27aaac59ecb 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java @@ -150,7 +150,7 @@ public void testFinalReduce() { SearchSourceBuilder source = new SearchSourceBuilder(); source.size(0); originalRequest.source(source); - TermsAggregationBuilder terms = new TermsAggregationBuilder("terms", ValueType.NUMERIC); + TermsAggregationBuilder terms = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.NUMERIC); terms.field("price"); terms.size(1); source.aggregation(terms); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index f58d061fb0113..de004eb6cdb6e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -128,7 +128,7 @@ public void setupCreateIndexRequestAndAliasValidator() { queryShardContext = new QueryShardContext(0, new IndexSettings(IndexMetaData.builder("test").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, () -> randomNonNegativeLong(), null, null, () -> true); + null, null, () -> randomNonNegativeLong(), null, null, () -> true, null); } private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) { diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 5c507ddec67d7..f0a4df2bbf12f 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -162,7 +162,7 @@ public void tearDown() throws Exception { private IndexService newIndexService(IndexModule module) throws IOException { return module.newIndexService(CREATE_INDEX, nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, null, indicesQueryCache, mapperRegistry, - new IndicesFieldDataCache(settings, listener), writableRegistry(), () -> false); + new IndicesFieldDataCache(settings, listener), writableRegistry(), () -> false, null); } public void testWrapperIsBound() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 2b33fa874ec0e..67d8be82b7aa8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -232,7 +232,7 @@ public void testTermQuery() { QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true); + xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); String date = "2015-10-12T14:10:55"; @@ -255,7 +255,7 @@ public void testRangeQuery() throws IOException { QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, () -> nowInMillis, null, null, () -> true); + null, null, () -> nowInMillis, null, null, () -> true, null); MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); String date1 = "2015-10-12T14:10:55"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index 4635c6f28ccf7..0ae583cef3f08 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -68,7 +68,7 @@ public void testTermQuery() { QueryShardContext queryShardContext = new QueryShardContext(0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, - null, null, null, null, null, null, () -> 0L, null, null, () -> true); + null, null, null, null, null, null, () -> 0L, null, null, () -> true, null); fieldNamesFieldType.setEnabled(true); Query termQuery = fieldNamesFieldType.termQuery("field_name", queryShardContext); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.CONTENT_TYPE, "field_name")), termQuery); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index bf0a0dffba743..b4a52d2d0cc53 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -78,6 +78,6 @@ private QueryShardContext createContext() { Predicate indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index"); return new QueryShardContext(0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, System::currentTimeMillis, null, indexNameMatcher, () -> true); + null, null, System::currentTimeMillis, null, indexNameMatcher, () -> true, null); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index d1a7ff06d3276..64880200547fa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -230,7 +230,7 @@ private QueryShardContext createContext() { .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true); + xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); } public void testDateRangeQueryUsingMappingFormat() { diff --git a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java index ed6b58390f645..dc05e40efd92b 100644 --- a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java @@ -424,7 +424,7 @@ public FactoryType compile(Script script, ScriptContext true); + null, scriptService, null, null, null, null, null, null, null, () -> true, null); String json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " + "\"match\" : { " + diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index c103bfc1a4d8b..99618982951bb 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -153,6 +153,6 @@ public static QueryShardContext createQueryShardContext(String indexUuid, String (mappedFieldType, idxName) -> mappedFieldType.fielddataBuilder(idxName).build(indexSettings, mappedFieldType, null, null, null), mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> nowInMillis, clusterAlias, null, () -> true); + null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index c43470ea3b21c..263a9eb95942f 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -41,7 +41,7 @@ public void testRewriteMissingField() throws Exception { IndexReader reader = new MultiReader(); QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), - null, new IndexSearcher(reader), null, null, null, () -> true); + null, new IndexSearcher(reader), null, null, null, () -> true, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); assertEquals(Relation.DISJOINT, range.getRelation(context)); } @@ -59,7 +59,7 @@ public void testRewriteMissingReader() throws Exception { new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), - null, null, null, null, null, () -> true); + null, null, null, null, null, () -> true, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // can't make assumptions on a missing reader, so it must return INTERSECT assertEquals(Relation.INTERSECTS, range.getRelation(context)); @@ -79,7 +79,7 @@ public void testRewriteEmptyReader() throws Exception { IndexReader reader = new MultiReader(); QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), - null, new IndexSearcher(reader), null, null, null, () -> true); + null, new IndexSearcher(reader), null, null, null, () -> true, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // no values -> DISJOINT assertEquals(Relation.DISJOINT, range.getRelation(context)); diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java index 48daf474d0ded..e9ccf3c7529f6 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -46,10 +46,11 @@ import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.InternalDerivative; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.subphase.ExplainPhase; import org.elasticsearch.search.fetch.subphase.highlight.CustomHighlighter; @@ -359,12 +360,17 @@ public List> getRescorers() { /** * Dummy test {@link AggregationBuilder} used to test registering aggregation builders. */ - private static class TestAggregationBuilder extends ValuesSourceAggregationBuilder { + private static class TestAggregationBuilder extends ValuesSourceAggregationBuilder { protected TestAggregationBuilder(TestAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new TestAggregationBuilder(this, factoriesBuilder, metaData); @@ -373,7 +379,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return null; } diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index 39126ae54778b..86f1cfbdd175e 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -657,7 +657,7 @@ public void testCanMatch() throws IOException, InterruptedException { new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()) - .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(0))); + .aggregation(new TermsAggregationBuilder("test").userValueTypeHint(ValueType.STRING).minDocCount(0))); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()) @@ -704,12 +704,12 @@ public void testCanRewriteToMatchNone() { assertFalse(SearchService.canRewriteToMatchNone(new SearchSourceBuilder())); assertFalse(SearchService.canRewriteToMatchNone(null)); assertFalse(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()) - .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(0)))); + .aggregation(new TermsAggregationBuilder("test").userValueTypeHint(ValueType.STRING).minDocCount(0)))); assertTrue(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new TermQueryBuilder("foo", "bar")))); assertTrue(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()) - .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(1)))); + .aggregation(new TermsAggregationBuilder("test").userValueTypeHint(ValueType.STRING).minDocCount(1)))); assertFalse(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()) - .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(1)) + .aggregation(new TermsAggregationBuilder("test").userValueTypeHint(ValueType.STRING).minDocCount(1)) .suggest(new SuggestBuilder()))); assertFalse(SearchService.canRewriteToMatchNone(new SearchSourceBuilder().query(new TermQueryBuilder("foo", "bar")) .suggest(new SuggestBuilder()))); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java index aa1ff6f55af82..a633aee5f3cc6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java @@ -168,7 +168,7 @@ public void testRewrite() throws IOException { assertTrue(((FiltersAggregationBuilder) rewritten).isKeyed()); // test sub-agg filter that does rewrite - original = new TermsAggregationBuilder("terms", ValueType.BOOLEAN) + original = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.BOOLEAN) .subAggregation( new FiltersAggregationBuilder("my-agg", new KeyedFilter("my-filter", new BoolQueryBuilder())) ); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java index fe6eb965ef06c..d5989ab0a3e0d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java @@ -444,7 +444,7 @@ public void testEmptyAggregation() throws Exception { public void testNoRangesInQuery() { try { client().prepareSearch("idx") - .addAggregation(geoDistance("geo_dist", new GeoPoint(52.3760, 4.894))) + .addAggregation(geoDistance("geo_dist", new GeoPoint(52.3760, 4.894)).field("location")) .get(); fail(); } catch (SearchPhaseExecutionException spee){ diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingTests.java index af8cfc01b3e97..9f63bb6c7a36d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingTests.java @@ -26,7 +26,7 @@ public class MissingTests extends BaseAggregationTestCase @Override protected TermsAggregationBuilder createTestAggregatorBuilder() { String name = randomAlphaOfLengthBetween(3, 20); - TermsAggregationBuilder factory = new TermsAggregationBuilder(name, null); + TermsAggregationBuilder factory = new TermsAggregationBuilder(name); String field = randomAlphaOfLengthBetween(3, 20); randomFieldOrScript(factory, field); if (randomBoolean()) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index dbea1d7f2729e..f74b23889f393 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -1632,7 +1632,7 @@ public void testWithTermsSubAggExecutionMode() throws Exception { .field("keyword"); return new CompositeAggregationBuilder("name", Collections.singletonList(terms)) .subAggregation( - new TermsAggregationBuilder("terms", ValueType.STRING) + new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING) .field("terms") .collectMode(mode) .subAggregation(new MaxAggregationBuilder("max").field("long")) @@ -1660,7 +1660,7 @@ public void testWithTermsSubAggExecutionMode() throws Exception { .field("keyword"); return new CompositeAggregationBuilder("name", Collections.singletonList(terms)) .subAggregation( - new TermsAggregationBuilder("terms", ValueType.STRING) + new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING) .field("terms") .collectMode(mode) .subAggregation(new MaxAggregationBuilder("max").field("long")) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java index 4afce4e5ff247..913d824399f37 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java @@ -97,7 +97,7 @@ public void testParseAndValidate() { QueryShardContext qsc = new QueryShardContext(0, new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, () -> now, null, null, () -> true); + null, null, () -> now, null, null, () -> true, null); DateFormatter formatter = DateFormatter.forPattern("dateOptionalTime"); DocValueFormat format = new DocValueFormat.DateTime(formatter, ZoneOffset.UTC, DateFieldMapper.Resolution.MILLISECONDS); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java index e3d1b931c71d5..1d49f871d2816 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java @@ -29,11 +29,16 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; + +import java.util.Arrays; +import java.util.List; + import static org.hamcrest.Matchers.containsString; public class NumericHistogramAggregatorTests extends AggregatorTestCase { @@ -100,6 +105,50 @@ public void testDoubles() throws Exception { } } + public void testDates() throws Exception { + List dataset = Arrays.asList( + "2019-11-01T01:07:45", + "2019-11-02T03:43:34", + "2019-11-03T04:11:00", + "2019-11-04T05:11:31", + "2019-11-05T08:24:05", + "2019-11-06T13:09:32", + "2019-11-07T13:47:43", + "2019-11-08T16:14:34", + "2019-11-09T17:09:50", + "2019-11-10T22:55:46"); + + String fieldName = "date_field"; + DateFieldMapper.Builder builder = new DateFieldMapper.Builder(fieldName); + DateFieldMapper.DateFieldType fieldType = builder.fieldType(); + fieldType.setName(fieldName); + fieldType.setHasDocValues(true); + + try (Directory dir = newDirectory(); + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), dir)) { + Document document = new Document(); + for (String date : dataset) { + if (frequently()) { + indexWriter.commit(); + } + + long instant = fieldType.parse(date); + document.add(new SortedNumericDocValuesField(fieldName, instant)); + indexWriter.addDocument(document); + document.clear(); + } + + HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") + .field(fieldName) + .interval(1000 * 60 * 60 * 24); + try (IndexReader reader = indexWriter.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertTrue(AggregationInspectionHelper.hasValue(histogram)); + } + } + } + public void testIrrationalInterval() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index 3665fcded7280..fe0695a98b0e4 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -20,12 +20,10 @@ package org.elasticsearch.search.aggregations.bucket.missing; import org.apache.lucene.document.BinaryDocValuesField; -import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -33,7 +31,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; @@ -47,11 +44,6 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; -import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram; -import org.elasticsearch.search.aggregations.metrics.InternalSum; -import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -61,7 +53,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -73,10 +64,6 @@ import static java.util.Collections.singletonMap; import static java.util.stream.Collectors.toList; import static org.elasticsearch.common.lucene.search.Queries.newMatchAllQuery; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.collection.IsEmptyCollection.empty; public class MissingAggregatorTests extends AggregatorTestCase { @@ -93,7 +80,7 @@ public void testMatchNoDocs() throws IOException { final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); fieldType.setName("field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .field(fieldType.name()); testCase( @@ -104,9 +91,9 @@ public void testMatchNoDocs() throws IOException { writer.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong()))); } }, - (InternalMissing missing) -> { - assertEquals(0, missing.getDocCount()); - assertFalse(AggregationInspectionHelper.hasValue(missing)); + internalMissing -> { + assertEquals(0, internalMissing.getDocCount()); + assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); }, singleton(fieldType) ); @@ -120,7 +107,7 @@ public void testMatchAllDocs() throws IOException { final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); anotherFieldType.setName("another_field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .field(aggFieldType.name()); testCase( @@ -131,9 +118,9 @@ public void testMatchAllDocs() throws IOException { writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); } }, - (InternalMissing missing) -> { - assertEquals(numDocs, missing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(missing)); + internalMissing -> { + assertEquals(numDocs, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, List.of(aggFieldType, anotherFieldType) ); @@ -145,7 +132,7 @@ public void testMatchSparse() throws IOException { final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); anotherFieldType.setName("another_field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .field(aggFieldType.name()); final int numDocs = randomIntBetween(100, 200); @@ -165,9 +152,9 @@ public void testMatchSparse() throws IOException { newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - (InternalMissing missing) -> { - assertEquals(finalDocsMissingAggField, missing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(missing)); + internalMissing -> { + assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, List.of(aggFieldType, anotherFieldType) ); @@ -184,7 +171,7 @@ public void testMatchSparseRangeField() throws IOException { final BytesRef encodedRange = rangeType.encodeRanges(singleton(range)); final BinaryDocValuesField encodedRangeField = new BinaryDocValuesField(aggFieldType.name(), encodedRange); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .field(aggFieldType.name()); final int numDocs = randomIntBetween(100, 200); @@ -204,110 +191,11 @@ public void testMatchSparseRangeField() throws IOException { newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - (InternalMissing missing) -> { - assertEquals(finalDocsMissingAggField, missing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(missing)); - }, - List.of(aggFieldType, anotherFieldType) - ); - } - - public void testSubAggregation() throws IOException { - final MappedFieldType missingAggFieldType = new KeywordFieldMapper.Builder("_name").fieldType(); - missingAggFieldType.setName("missing_agg_field"); - missingAggFieldType.setHasDocValues(true); - final MappedFieldType sumAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); - sumAggFieldType.setName("sum_agg_field"); - - final MissingAggregationBuilder builder = new MissingAggregationBuilder("missing", null) - .field(missingAggFieldType.name()) - .subAggregation( - new SumAggregationBuilder("sum") - .field(sumAggFieldType.name()) - ); - - final int numDocs = randomIntBetween(10, 20); - int docsMissingAggField = 0; - long expectedSum = 0; - final List> docs = new ArrayList<>(); - for (int i = 0; i < numDocs; i++) { - final Set doc = new HashSet<>(); - final long sumFieldValue = randomLongBetween(0, 1000); - doc.add(new SortedNumericDocValuesField(sumAggFieldType.name(), sumFieldValue)); - if (randomBoolean()) { - doc.add(new SortedDocValuesField(missingAggFieldType.name(), new BytesRef(randomUnicodeOfLengthBetween(2, 20)))); - } else { - docsMissingAggField++; - expectedSum += sumFieldValue; - } - docs.add(doc); - } - final int finalDocsMissingAggField = docsMissingAggField; - final long finalExpectedSum = expectedSum; - - testCase( - newMatchAllQuery(), - builder, - writer -> writer.addDocuments(docs), - (InternalMissing internalMissing) -> { + internalMissing -> { assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); - - assertThat(internalMissing.getAggregations().asList(), not(empty())); - final InternalSum internalSum = internalMissing.getAggregations().get("sum"); - assertEquals(finalExpectedSum, internalSum.getValue(), 0d); - assertTrue(AggregationInspectionHelper.hasValue(internalSum)); - }, - List.of(missingAggFieldType, sumAggFieldType) - ); - } - - public void testEmptyBucket() throws IOException { - final MappedFieldType histoAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); - histoAggFieldType.setName("histo_agg_field"); - final MappedFieldType missingAggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); - missingAggFieldType.setName("missing_agg_field"); - - final HistogramAggregationBuilder builder = new HistogramAggregationBuilder("histo") - .field(histoAggFieldType.name()) - .interval(1) - .minDocCount(0) - .subAggregation( - new MissingAggregationBuilder("missing", null) - .field(missingAggFieldType.name()) - ); - - testCaseWithReduce( - newMatchAllQuery(), - builder, - writer -> { - writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 0))); - writer.addDocument(singleton(new SortedNumericDocValuesField(histoAggFieldType.name(), 2))); }, - (InternalHistogram histogram) -> { - assertThat(histogram.getBuckets(), hasSize(3)); - - { - assertThat(histogram.getBuckets().get(0), notNullValue()); - final InternalMissing missing = histogram.getBuckets().get(0).getAggregations().get("missing"); - assertEquals(1, missing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(missing)); - } - { - assertThat(histogram.getBuckets().get(1), notNullValue()); - final InternalMissing missing = histogram.getBuckets().get(1).getAggregations().get("missing"); - assertEquals(0, missing.getDocCount()); - assertFalse(AggregationInspectionHelper.hasValue(missing)); - } - { - assertThat(histogram.getBuckets().get(2), notNullValue()); - final InternalMissing missing = histogram.getBuckets().get(2).getAggregations().get("missing"); - assertEquals(1, missing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(missing)); - } - }, - List.of(histoAggFieldType, missingAggFieldType), - true + List.of(aggFieldType, anotherFieldType) ); } @@ -316,7 +204,7 @@ public void testUnmappedWithoutMissingParam() throws IOException { final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); aggFieldType.setName("agg_field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .field("unknown_field"); testCase( @@ -327,9 +215,9 @@ public void testUnmappedWithoutMissingParam() throws IOException { writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong()))); } }, - (InternalMissing missing) -> { - assertEquals(numDocs, missing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(missing)); + internalMissing -> { + assertEquals(numDocs, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, singleton(aggFieldType) ); @@ -340,7 +228,7 @@ public void testUnmappedWithMissingParam() throws IOException { final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); aggFieldType.setName("agg_field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .field("unknown_field") .missing(randomLong()); @@ -352,44 +240,14 @@ public void testUnmappedWithMissingParam() throws IOException { writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong()))); } }, - (InternalMissing missing) -> { - assertEquals(0, missing.getDocCount()); - assertFalse(AggregationInspectionHelper.hasValue(missing)); + internalMissing -> { + assertEquals(0, internalMissing.getDocCount()); + assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); }, singleton(aggFieldType) ); } - public void testSingleValuedFieldPartiallyUnmapped() throws IOException { - final int numDocs = randomIntBetween(10, 20); - - final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); - fieldType.setName("field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) - .field(fieldType.name()); - - try (Directory mappedDirectory = newDirectory(); Directory unmappedDirectory = newDirectory()) { - try (RandomIndexWriter mappedWriter = new RandomIndexWriter(random(), mappedDirectory)) { - for (int i = 0; i < numDocs; i++) { - mappedWriter.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong()))); - } - } - - new RandomIndexWriter(random(), unmappedDirectory).close(); - - try (IndexReader mappedReader = DirectoryReader.open(mappedDirectory); - IndexReader unmappedReader = DirectoryReader.open(unmappedDirectory); - MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) { - - final IndexSearcher searcher = newSearcher(multiReader, true, true); - - final InternalMissing internalMissing = searchAndReduce(searcher, newMatchAllQuery(), builder, fieldType); - assertEquals(0, internalMissing.getDocCount()); - assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); - } - } - } - public void testMissingParam() throws IOException { final int numDocs = randomIntBetween(10, 20); @@ -398,7 +256,7 @@ public void testMissingParam() throws IOException { final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); anotherFieldType.setName("another_field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .field(aggFieldType.name()) .missing(randomLong()); @@ -410,9 +268,9 @@ public void testMissingParam() throws IOException { writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); } }, - (InternalMissing missing) -> { - assertEquals(0, missing.getDocCount()); - assertFalse(AggregationInspectionHelper.hasValue(missing)); + internalMissing -> { + assertEquals(0, internalMissing.getDocCount()); + assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); }, List.of(aggFieldType, anotherFieldType) ); @@ -424,7 +282,7 @@ public void testMultiValuedField() throws IOException { final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); anotherFieldType.setName("another_field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .field(aggFieldType.name()); final int numDocs = randomIntBetween(100, 200); @@ -448,9 +306,9 @@ public void testMultiValuedField() throws IOException { newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - (InternalMissing missing) -> { - assertEquals(finalDocsMissingAggField, missing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(missing)); + internalMissing -> { + assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, List.of(aggFieldType, anotherFieldType) ); @@ -470,7 +328,7 @@ private void valueScriptTestCase(Script script) throws IOException { final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); anotherFieldType.setName("another_field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .field(aggFieldType.name()) .script(script); @@ -491,9 +349,9 @@ private void valueScriptTestCase(Script script) throws IOException { newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - (InternalMissing missing) -> { - assertEquals(finalDocsMissingField, missing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(missing)); + internalMissing -> { + assertEquals(finalDocsMissingField, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, List.of(aggFieldType, anotherFieldType) ); @@ -514,7 +372,7 @@ private void fieldScriptTestCase(Script script, long threshold) throws IOExcepti final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); aggFieldType.setName("agg_field"); - final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name") .script(script); final int numDocs = randomIntBetween(100, 200); @@ -537,32 +395,29 @@ private void fieldScriptTestCase(Script script, long threshold) throws IOExcepti newMatchAllQuery(), builder, writer -> writer.addDocuments(docs), - (InternalMissing missing) -> { - assertEquals(finalDocsBelowThreshold, missing.getDocCount()); - assertTrue(AggregationInspectionHelper.hasValue(missing)); + internalMissing -> { + assertEquals(finalDocsBelowThreshold, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, singleton(aggFieldType) ); } - private void testCase( - Query query, - B builder, - CheckedConsumer writeIndex, - Consumer verify, - Collection fieldTypes) throws IOException { - + private void testCase(Query query, + MissingAggregationBuilder builder, + CheckedConsumer writeIndex, + Consumer verify, + Collection fieldTypes) throws IOException { testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, false); testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, true); } - private void testCaseWithReduce( - Query query, - B builder, - CheckedConsumer writeIndex, - Consumer verify, - Collection fieldTypes, - boolean reduced) throws IOException { + private void testCaseWithReduce(Query query, + MissingAggregationBuilder builder, + CheckedConsumer writeIndex, + Consumer verify, + Collection fieldTypes, + boolean reduced) throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { @@ -572,20 +427,20 @@ private void testC try (IndexReader indexReader = DirectoryReader.open(directory)) { final IndexSearcher indexSearcher = newSearcher(indexReader, true, true); final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]); - final I result; + final InternalMissing missing; if (reduced) { - result = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); + missing = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); } else { - result = search(indexSearcher, query, builder, fieldTypesArray); + missing = search(indexSearcher, query, builder, fieldTypesArray); } - verify.accept(result); + verify.accept(missing); } } } @Override protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { - return new MissingAggregationBuilder("_name", null) + return new MissingAggregationBuilder("_name") .field(fieldName); } @@ -596,7 +451,9 @@ protected List getSupportedValuesSourceTypes() { CoreValuesSourceType.BYTES, CoreValuesSourceType.GEOPOINT, CoreValuesSourceType.RANGE, - CoreValuesSourceType.HISTOGRAM + CoreValuesSourceType.IP, + CoreValuesSourceType.BOOLEAN, + CoreValuesSourceType.DATE ); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index a94c5c34a7a77..a2f4b3d096659 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -416,7 +416,7 @@ public void testNestedOrdering() throws IOException { fieldType2.setHasDocValues(true); fieldType2.setName("author"); - TermsAggregationBuilder termsBuilder = new TermsAggregationBuilder("authors", ValueType.STRING) + TermsAggregationBuilder termsBuilder = new TermsAggregationBuilder("authors").userValueTypeHint(ValueType.STRING) .field("author").order(BucketOrder.aggregation("chapters>num_pages.value", true)); NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder("chapters", "nested_chapters"); MaxAggregationBuilder maxAgg = new MaxAggregationBuilder("num_pages").field("num_pages"); @@ -465,7 +465,7 @@ public void testNestedOrdering() throws IOException { assertEquals(70, (int) numPages.getValue()); // reverse order: - termsBuilder = new TermsAggregationBuilder("authors", ValueType.STRING) + termsBuilder = new TermsAggregationBuilder("authors").userValueTypeHint(ValueType.STRING) .field("author").order(BucketOrder.aggregation("chapters>num_pages.value", false)); nestedBuilder = new NestedAggregationBuilder("chapters", "nested_chapters"); maxAgg = new MaxAggregationBuilder("num_pages").field("num_pages"); @@ -554,7 +554,7 @@ public void testNestedOrdering_random() throws IOException { fieldType2.setHasDocValues(true); fieldType2.setName("author"); - TermsAggregationBuilder termsBuilder = new TermsAggregationBuilder("authors", ValueType.STRING) + TermsAggregationBuilder termsBuilder = new TermsAggregationBuilder("authors").userValueTypeHint(ValueType.STRING) .size(books.size()).field("author") .order(BucketOrder.compound(BucketOrder.aggregation("chapters>num_pages.value", true), BucketOrder.key(true))); NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder("chapters", "nested_chapters"); @@ -646,8 +646,10 @@ public void testPreGetChildLeafCollectors() throws IOException { iw.commit(); } try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) { - TermsAggregationBuilder valueBuilder = new TermsAggregationBuilder("value", ValueType.STRING).field("value"); - TermsAggregationBuilder keyBuilder = new TermsAggregationBuilder("key", ValueType.STRING).field("key"); + TermsAggregationBuilder valueBuilder = new TermsAggregationBuilder("value").userValueTypeHint(ValueType.STRING) + .field("value"); + TermsAggregationBuilder keyBuilder = new TermsAggregationBuilder("key").userValueTypeHint(ValueType.STRING) + .field("key"); keyBuilder.subAggregation(valueBuilder); NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder(NESTED_AGG, "nested_field"); nestedBuilder.subAggregation(keyBuilder); @@ -763,7 +765,7 @@ public void testNestedWithPipeline() throws IOException { } try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) { NestedAggregationBuilder nestedBuilder = new NestedAggregationBuilder(NESTED_AGG, NESTED_OBJECT) - .subAggregation(new TermsAggregationBuilder("terms", ValueType.NUMERIC).field(VALUE_FIELD_NAME) + .subAggregation(new TermsAggregationBuilder("terms").field(VALUE_FIELD_NAME).userValueTypeHint(ValueType.NUMERIC) .subAggregation(new MaxAggregationBuilder(MAX_AGG_NAME).field(VALUE_FIELD_NAME)) .subAggregation(new BucketScriptPipelineAggregationBuilder("bucketscript", Collections.singletonMap("_value", MAX_AGG_NAME), diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java index 2746e69be29de..2fb281170b732 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java @@ -224,8 +224,8 @@ public void testKeywordField() { () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { iw.addDocument(singleton(new SortedSetDocValuesField("string", new BytesRef("foo")))); }, range -> fail("Should have thrown exception"), fieldType)); - // I believe this error is coming from improperly parsing the range, not the field. - assertEquals("For input string: \"2015-11-13\"", e.getMessage()); + assertEquals("Field [not_a_number] of type [keyword(indexed,tokenized)] is not supported for aggregation [date_range]", + e.getMessage()); } public void testBadMissingField() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index 2b3571ffaac85..092a0319c9c21 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -238,7 +238,7 @@ public void testUnsupportedType() { () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { iw.addDocument(singleton(new SortedSetDocValuesField("string", new BytesRef("foo")))); }, range -> fail("Should have thrown exception"), fieldType)); - assertEquals(e.getMessage(), "Expected numeric type on field [not_a_number], but got [keyword]"); + assertEquals("Field [not_a_number] of type [keyword(indexed,tokenized)] is not supported for aggregation [range]", e.getMessage()); } public void testBadMissingField() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java index 5c29aabfa9dc4..d9ebbf633f242 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java @@ -201,7 +201,7 @@ private void testCase(IndexSearcher indexSearcher, MappedFieldType genreFieldTyp .executionHint(executionHint) .maxDocsPerValue(maxDocsPerValue) .shardSize(shardSize) - .subAggregation(new TermsAggregationBuilder("terms", null).field("id")); + .subAggregation(new TermsAggregationBuilder("terms").field("id")); InternalSampler result = search(indexSearcher, query, builder, genreFieldType, idFieldType); verify.accept(result); @@ -224,7 +224,7 @@ public void testDiversifiedSampler_noDocs() throws Exception { DiversifiedAggregationBuilder builder = new DiversifiedAggregationBuilder("_name") .field(genreFieldType.name()) - .subAggregation(new TermsAggregationBuilder("terms", null).field("id")); + .subAggregation(new TermsAggregationBuilder("terms").field("id")); InternalSampler result = search(indexSearcher, new MatchAllDocsQuery(), builder, genreFieldType, idFieldType); Terms terms = result.getAggregations().get("terms"); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java index aa35aab052b2c..9b97b0e1ee8e6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java @@ -48,12 +48,10 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregatorFactory.ExecutionMode; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.junit.Before; @@ -82,7 +80,7 @@ public void setUpTest() throws Exception { @Override protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { - return new SignificantTermsAggregationBuilder("foo", ValueType.STRING).field(fieldName); + return new SignificantTermsAggregationBuilder("foo").field(fieldName); } @Override @@ -127,14 +125,14 @@ public void testSignificance() throws IOException { try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { addMixedTextDocs(textFieldType, w); - SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text", null).field("text"); + SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text").field("text"); sigAgg.executionHint(randomExecutionHint()); if (randomBoolean()) { // Use a background filter which just happens to be same scope as whole-index. sigAgg.backgroundFilter(QueryBuilders.termsQuery("text", "common")); } - SignificantTermsAggregationBuilder sigNumAgg = new SignificantTermsAggregationBuilder("sig_number", null).field("long_field"); + SignificantTermsAggregationBuilder sigNumAgg = new SignificantTermsAggregationBuilder("sig_number").field("long_field"); sigNumAgg.executionHint(randomExecutionHint()); try (IndexReader reader = DirectoryReader.open(w)) { @@ -226,7 +224,7 @@ public void testNumericSignificance() throws IOException { w.addDocument(doc); } - SignificantTermsAggregationBuilder sigNumAgg = new SignificantTermsAggregationBuilder("sig_number", null).field("long_field"); + SignificantTermsAggregationBuilder sigNumAgg = new SignificantTermsAggregationBuilder("sig_number").field("long_field"); sigNumAgg.executionHint(randomExecutionHint()); try (IndexReader reader = DirectoryReader.open(w)) { @@ -268,7 +266,7 @@ public void testUnmapped() throws IOException { addMixedTextDocs(textFieldType, w); // Attempt aggregation on unmapped field - SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text", null).field("unmapped_field"); + SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text").field("unmapped_field"); sigAgg.executionHint(randomExecutionHint()); try (IndexReader reader = DirectoryReader.open(w)) { @@ -315,12 +313,12 @@ public void testRangeField() throws IOException { } // Attempt aggregation on range field - SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text", null).field(fieldName); + SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text").field(fieldName); sigAgg.executionHint(randomExecutionHint()); try (IndexReader reader = DirectoryReader.open(w)) { IndexSearcher indexSearcher = newIndexSearcher(reader); - expectThrows(AggregationExecutionException.class, () -> createAggregator(sigAgg, indexSearcher, fieldType)); + expectThrows(IllegalArgumentException.class, () -> createAggregator(sigAgg, indexSearcher, fieldType)); } } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java new file mode 100644 index 0000000000000..31859a7a4fa04 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java @@ -0,0 +1,176 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.terms; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.common.Numbers; +import org.elasticsearch.index.mapper.BinaryFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.support.ValueType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.equalTo; + +public class BinaryTermsAggregatorTests extends AggregatorTestCase { + private static final String BINARY_FIELD = "binary"; + + private static final List dataset; + static { + List d = new ArrayList<>(45); + for (int i = 0; i < 10; i++) { + for (int j = 0; j < i; j++) { + d.add((long) i); + } + } + dataset = d; + } + + public void testMatchNoDocs() throws IOException { + testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(BINARY_FIELD), + agg -> assertEquals(0, agg.getBuckets().size()), ValueType.STRING + ); + } + + public void testMatchAllDocs() throws IOException { + Query query = new MatchAllDocsQuery(); + + testBothCases(query, dataset, + aggregation -> aggregation.field(BINARY_FIELD), + agg -> { + assertEquals(9, agg.getBuckets().size()); + for (int i = 0; i < 9; i++) { + StringTerms.Bucket bucket = (StringTerms.Bucket) agg.getBuckets().get(i); + byte[] bytes = Numbers.longToBytes(9L - i); + String bytesAsString = (String) DocValueFormat.BINARY.format(new BytesRef(bytes)); + assertThat(bucket.getKey(), equalTo(bytesAsString)); + assertThat(bucket.getDocCount(), equalTo(9L - i)); + } + }, null); + } + + public void testBadIncludeExclude() throws IOException { + IncludeExclude includeExclude = new IncludeExclude(new RegExp("foo"), null); + + // Make sure the include/exclude fails regardless of how the user tries to type hint the agg + AggregationExecutionException e = expectThrows(AggregationExecutionException.class, + () -> testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(BINARY_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), + agg -> fail("test should have failed with exception"), null // default, no hint + )); + assertThat(e.getMessage(), equalTo("Aggregation [_name] cannot support regular expression style include/exclude settings as " + + "they can only be applied to string fields. Use an array of values for include/exclude clauses")); + + e = expectThrows(AggregationExecutionException.class, + () -> testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(BINARY_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), + agg -> fail("test should have failed with exception"), ValueType.STRING // string type hint + )); + assertThat(e.getMessage(), equalTo("Aggregation [_name] cannot support regular expression style include/exclude settings as " + + "they can only be applied to string fields. Use an array of values for include/exclude clauses")); + + e = expectThrows(AggregationExecutionException.class, () -> testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(BINARY_FIELD).includeExclude(includeExclude), + agg -> fail("test should have failed with exception"), ValueType.NUMERIC // numeric type hint + )); + assertThat(e.getMessage(), equalTo("Aggregation [_name] cannot support regular expression style include/exclude settings as " + + "they can only be applied to string fields. Use an array of values for include/exclude clauses")); + } + + private void testSearchCase(Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + executeTestCase(false, query, dataset, configure, verify, valueType); + } + + private void testSearchAndReduceCase(Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + executeTestCase(true, query, dataset, configure, verify, valueType); + } + + private void testBothCases(Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + testSearchCase(query, dataset, configure, verify, valueType); + testSearchAndReduceCase(query, dataset, configure, verify, valueType); + } + + private void executeTestCase(boolean reduced, Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + for (Long value : dataset) { + if (frequently()) { + indexWriter.commit(); + } + + document.add(new BinaryFieldMapper.CustomBinaryDocValuesField(BINARY_FIELD, Numbers.longToBytes(value))); + indexWriter.addDocument(document); + document.clear(); + } + } + + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name"); + if (valueType != null) { + aggregationBuilder.userValueTypeHint(valueType); + } + if (configure != null) { + configure.accept(aggregationBuilder); + } + + MappedFieldType binaryFieldType = new BinaryFieldMapper.Builder(BINARY_FIELD).fieldType(); + binaryFieldType.setName(BINARY_FIELD); + binaryFieldType.setHasDocValues(true); + + InternalMappedTerms rareTerms; + if (reduced) { + rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, binaryFieldType); + } else { + rareTerms = search(indexSearcher, query, aggregationBuilder, binaryFieldType); + } + verify.accept(rareTerms); + } + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java new file mode 100644 index 0000000000000..098e94d3d4450 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java @@ -0,0 +1,161 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.terms; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.support.ValueType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.equalTo; + +public class KeywordTermsAggregatorTests extends AggregatorTestCase { + private static final String KEYWORD_FIELD = "keyword"; + + private static final List dataset; + static { + List d = new ArrayList<>(45); + for (int i = 0; i < 10; i++) { + for (int j = 0; j < i; j++) { + d.add(String.valueOf(i)); + } + } + dataset = d; + } + + public void testMatchNoDocs() throws IOException { + testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(KEYWORD_FIELD), + agg -> assertEquals(0, agg.getBuckets().size()), null // without type hint + ); + + testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(KEYWORD_FIELD), + agg -> assertEquals(0, agg.getBuckets().size()), ValueType.STRING // with type hint + ); + } + + public void testMatchAllDocs() throws IOException { + Query query = new MatchAllDocsQuery(); + + testBothCases(query, dataset, + aggregation -> aggregation.field(KEYWORD_FIELD), + agg -> { + assertEquals(9, agg.getBuckets().size()); + for (int i = 0; i < 9; i++) { + StringTerms.Bucket bucket = (StringTerms.Bucket) agg.getBuckets().get(i); + assertThat(bucket.getKey(), equalTo(String.valueOf(9L - i))); + assertThat(bucket.getDocCount(), equalTo(9L - i)); + } + }, null // without type hint + ); + + testBothCases(query, dataset, + aggregation -> aggregation.field(KEYWORD_FIELD), + agg -> { + assertEquals(9, agg.getBuckets().size()); + for (int i = 0; i < 9; i++) { + StringTerms.Bucket bucket = (StringTerms.Bucket) agg.getBuckets().get(i); + assertThat(bucket.getKey(), equalTo(String.valueOf(9L - i))); + assertThat(bucket.getDocCount(), equalTo(9L - i)); + } + }, ValueType.STRING // with type hint + ); + } + + private void testSearchCase(Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + executeTestCase(false, query, dataset, configure, verify, valueType); + } + + private void testSearchAndReduceCase(Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + executeTestCase(true, query, dataset, configure, verify, valueType); + } + + private void testBothCases(Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + testSearchCase(query, dataset, configure, verify, valueType); + testSearchAndReduceCase(query, dataset, configure, verify, valueType); + } + + private void executeTestCase(boolean reduced, Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + for (String value : dataset) { + if (frequently()) { + indexWriter.commit(); + } + + document.add(new SortedSetDocValuesField(KEYWORD_FIELD, new BytesRef(value))); + indexWriter.addDocument(document); + document.clear(); + } + } + + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name"); + if (valueType != null) { + aggregationBuilder.userValueTypeHint(valueType); + } + if (configure != null) { + configure.accept(aggregationBuilder); + } + + MappedFieldType keywordFieldType = new KeywordFieldMapper.KeywordFieldType(); + keywordFieldType.setName(KEYWORD_FIELD); + keywordFieldType.setHasDocValues(true); + + InternalMappedTerms rareTerms; + if (reduced) { + rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, keywordFieldType); + } else { + rareTerms = search(indexSearcher, query, aggregationBuilder, keywordFieldType); + } + verify.accept(rareTerms); + } + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java new file mode 100644 index 0000000000000..b0790b9fa0413 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java @@ -0,0 +1,190 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.terms; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.search.aggregations.AggregationExecutionException; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.support.ValueType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.equalTo; + +public class NumericTermsAggregatorTests extends AggregatorTestCase { + private static final String LONG_FIELD = "long"; + + private static final List dataset; + static { + List d = new ArrayList<>(45); + for (int i = 0; i < 10; i++) { + for (int j = 0; j < i; j++) { + d.add((long) i); + } + } + dataset = d; + } + + public void testMatchNoDocs() throws IOException { + + testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(LONG_FIELD), + agg -> assertEquals(0, agg.getBuckets().size()), null // without type hint + ); + + testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(LONG_FIELD), + agg -> assertEquals(0, agg.getBuckets().size()), ValueType.NUMERIC // with type hint + ); + } + + public void testMatchAllDocs() throws IOException { + Query query = new MatchAllDocsQuery(); + + testBothCases(query, dataset, + aggregation -> aggregation.field(LONG_FIELD), + agg -> { + assertEquals(9, agg.getBuckets().size()); + for (int i = 0; i < 9; i++) { + LongTerms.Bucket bucket = (LongTerms.Bucket) agg.getBuckets().get(i); + assertThat(bucket.getKey(), equalTo(9L - i)); + assertThat(bucket.getDocCount(), equalTo(9L - i)); + } + }, null //without type hint + ); + + testBothCases(query, dataset, + aggregation -> aggregation.field(LONG_FIELD), + agg -> { + assertEquals(9, agg.getBuckets().size()); + for (int i = 0; i < 9; i++) { + LongTerms.Bucket bucket = (LongTerms.Bucket) agg.getBuckets().get(i); + assertThat(bucket.getKey(), equalTo(9L - i)); + assertThat(bucket.getDocCount(), equalTo(9L - i)); + } + }, ValueType.NUMERIC //with type hint + ); + } + + public void testBadIncludeExclude() throws IOException { + IncludeExclude includeExclude = new IncludeExclude(new RegExp("foo"), null); + + // Numerics don't support any regex include/exclude, so should fail no matter what we do + + AggregationExecutionException e = expectThrows(AggregationExecutionException.class, + () -> testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(LONG_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), + agg -> fail("test should have failed with exception"), null + )); + assertThat(e.getMessage(), equalTo("Aggregation [_name] cannot support regular expression style " + + "include/exclude settings as they can only be applied to string fields. Use an array of numeric " + + "values for include/exclude clauses used to filter numeric fields")); + + e = expectThrows(AggregationExecutionException.class, + () -> testBothCases(new MatchNoDocsQuery(), dataset, + aggregation -> aggregation.field(LONG_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), + agg -> fail("test should have failed with exception"), ValueType.NUMERIC // with type hint + )); + assertThat(e.getMessage(), equalTo("Aggregation [_name] cannot support regular expression style " + + "include/exclude settings as they can only be applied to string fields. Use an array of numeric " + + "values for include/exclude clauses used to filter numeric fields")); + + } + + private void testSearchCase(Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + executeTestCase(false, query, dataset, configure, verify, valueType); + } + + private void testSearchAndReduceCase(Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + executeTestCase(true, query, dataset, configure, verify, valueType); + } + + private void testBothCases(Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + testSearchCase(query, dataset, configure, verify, valueType); + testSearchAndReduceCase(query, dataset, configure, verify, valueType); + } + + private void executeTestCase(boolean reduced, Query query, List dataset, + Consumer configure, + Consumer verify, ValueType valueType) throws IOException { + + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + for (Long value : dataset) { + if (frequently()) { + indexWriter.commit(); + } + + document.add(new SortedNumericDocValuesField(LONG_FIELD, value)); + document.add(new LongPoint(LONG_FIELD, value)); + indexWriter.addDocument(document); + document.clear(); + } + } + + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name"); + if (valueType != null) { + aggregationBuilder.userValueTypeHint(valueType); + } + if (configure != null) { + configure.accept(aggregationBuilder); + } + + MappedFieldType longFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + longFieldType.setName(LONG_FIELD); + longFieldType.setHasDocValues(true); + + InternalMappedTerms rareTerms; + if (reduced) { + rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, longFieldType); + } else { + rareTerms = search(indexSearcher, query, aggregationBuilder, longFieldType); + } + verify.accept(rareTerms); + } + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java index 94bb43f3db62c..1dffa9b768d9f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java @@ -52,7 +52,6 @@ import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; @@ -273,7 +272,8 @@ public void testUnmapped() throws Exception { ValueType[] valueTypes = new ValueType[]{ValueType.STRING, ValueType.LONG}; String[] fieldNames = new String[]{"string", "long"}; for (int i = 0; i < fieldNames.length; i++) { - RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name", valueTypes[i]) + RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name") + .userValueTypeHint(valueTypes[i]) .field(fieldNames[i]); Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1, fieldType2); aggregator.preCollection(); @@ -308,9 +308,9 @@ public void testRangeField() throws Exception { try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { IndexSearcher indexSearcher = newIndexSearcher(indexReader); - RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name", null) + RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name") .field("field"); - expectThrows(AggregationExecutionException.class, + expectThrows(IllegalArgumentException.class, () -> createAggregator(aggregationBuilder, indexSearcher, fieldType)); } } @@ -322,7 +322,8 @@ public void testNestedTerms() throws IOException { Query query = new MatchAllDocsQuery(); testBothCases(query, dataset, aggregation -> { - TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms", ValueType.STRING).field(KEYWORD_FIELD); + TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms").userValueTypeHint(ValueType.STRING) + .field(KEYWORD_FIELD); aggregation.field(LONG_FIELD).maxDocCount(1).subAggregation(terms); }, agg -> { @@ -340,7 +341,8 @@ public void testNestedTerms() throws IOException { ); testBothCases(query, dataset, aggregation -> { - TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms", ValueType.STRING).field(KEYWORD_FIELD); + TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms").userValueTypeHint(ValueType.STRING) + .field(KEYWORD_FIELD); aggregation.field(KEYWORD_FIELD).maxDocCount(1).subAggregation(terms); }, agg -> { @@ -375,10 +377,12 @@ public void testGlobalAggregationWithScore() throws IOException { Aggregator.SubAggCollectionMode collectionMode = randomFrom(Aggregator.SubAggCollectionMode.values()); GlobalAggregationBuilder globalBuilder = new GlobalAggregationBuilder("global") .subAggregation( - new RareTermsAggregationBuilder("terms", ValueType.STRING) + new RareTermsAggregationBuilder("terms") + .userValueTypeHint(ValueType.STRING) .field("keyword") .subAggregation( - new RareTermsAggregationBuilder("sub_terms", ValueType.STRING) + new RareTermsAggregationBuilder("sub_terms") + .userValueTypeHint(ValueType.STRING) .field("keyword") .subAggregation( new TopHitsAggregationBuilder("top_hits") @@ -422,7 +426,8 @@ public void testWithNestedAggregations() throws IOException { indexWriter.commit(); NestedAggregationBuilder nested = new NestedAggregationBuilder("nested", "nested_object") - .subAggregation(new RareTermsAggregationBuilder("terms", ValueType.LONG) + .subAggregation(new RareTermsAggregationBuilder("terms") + .userValueTypeHint(ValueType.LONG) .field("nested_value") .maxDocCount(1) ); @@ -455,7 +460,8 @@ public void testWithNestedScoringAggregations() throws IOException { indexWriter.commit(); for (boolean withScore : new boolean[]{true, false}) { NestedAggregationBuilder nested = new NestedAggregationBuilder("nested", "nested_object") - .subAggregation(new RareTermsAggregationBuilder("terms", ValueType.LONG) + .subAggregation(new RareTermsAggregationBuilder("terms") + .userValueTypeHint(ValueType.LONG) .field("nested_value") .maxDocCount(2) .subAggregation( @@ -580,7 +586,8 @@ private void executeTestCase(boolean reduced, Query query, List dataset, try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = newIndexSearcher(indexReader); - RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name", valueType); + RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name"); + aggregationBuilder.userValueTypeHint(valueType); if (configure != null) { configure.accept(aggregationBuilder); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 514f7549e757d..e48f6df687ef2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -127,13 +127,16 @@ protected A createAggregator(AggregationBuilder aggregati @Override protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { - return new TermsAggregationBuilder("foo", ValueType.STRING).field(fieldName); + return new TermsAggregationBuilder("foo").field(fieldName); } @Override protected List getSupportedValuesSourceTypes() { return List.of(CoreValuesSourceType.NUMERIC, - CoreValuesSourceType.BYTES); + CoreValuesSourceType.BYTES, + CoreValuesSourceType.IP, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN); } public void testGlobalOrdinalsExecutionHint() throws Exception { @@ -146,7 +149,7 @@ public void testGlobalOrdinalsExecutionHint() throws Exception { // We do not use LuceneTestCase.newSearcher because we need a DirectoryReader IndexSearcher indexSearcher = new IndexSearcher(indexReader); - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.STRING) .field("string") .collectMode(Aggregator.SubAggCollectionMode.BREADTH_FIRST); MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType(); @@ -198,7 +201,8 @@ public void testSimple() throws Exception { try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { IndexSearcher indexSearcher = newIndexSearcher(indexReader); for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) { - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .userValueTypeHint(ValueType.STRING) .executionHint(executionMode.toString()) .field("string") .order(BucketOrder.key(true)); @@ -269,7 +273,8 @@ public void testStringIncludeExclude() throws Exception { fieldType.setHasDocValues(true); String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString(); - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .includeExclude(new IncludeExclude("val00.+", null)) .field("mv_field") @@ -307,7 +312,7 @@ public void testStringIncludeExclude() throws Exception { MappedFieldType fieldType2 = new KeywordFieldMapper.KeywordFieldType(); fieldType2.setName("sv_field"); fieldType2.setHasDocValues(true); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .includeExclude(new IncludeExclude("val00.+", null)) .field("sv_field") @@ -331,7 +336,7 @@ public void testStringIncludeExclude() throws Exception { assertEquals(1L, result.getBuckets().get(4).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue((InternalTerms)result)); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .includeExclude(new IncludeExclude("val00.+", "(val000|val001)")) .field("mv_field") @@ -361,7 +366,7 @@ public void testStringIncludeExclude() throws Exception { assertEquals(1L, result.getBuckets().get(7).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue((InternalTerms)result)); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .includeExclude(new IncludeExclude(null, "val00.+")) .field("mv_field") @@ -378,7 +383,7 @@ public void testStringIncludeExclude() throws Exception { assertEquals(1L, result.getBuckets().get(1).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue((InternalTerms)result)); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .includeExclude(new IncludeExclude(new String[]{"val000", "val010"}, null)) .field("mv_field") @@ -395,7 +400,7 @@ public void testStringIncludeExclude() throws Exception { assertEquals(1L, result.getBuckets().get(1).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue((InternalTerms)result)); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .includeExclude(new IncludeExclude(null, new String[]{"val001", "val002", "val003", "val004", "val005", "val006", "val007", "val008", "val009", "val011"})) @@ -451,7 +456,8 @@ public void testNumericIncludeExclude() throws Exception { fieldType.setHasDocValues(true ); String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString(); - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .userValueTypeHint(ValueType.LONG) .executionHint(executionHint) .includeExclude(new IncludeExclude(new long[]{0, 5}, null)) .field("long_field") @@ -468,7 +474,7 @@ public void testNumericIncludeExclude() throws Exception { assertEquals(1L, result.getBuckets().get(1).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue((InternalTerms)result)); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.LONG) .executionHint(executionHint) .includeExclude(new IncludeExclude(null, new long[]{0, 5})) .field("long_field") @@ -492,7 +498,7 @@ public void testNumericIncludeExclude() throws Exception { fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); fieldType.setName("double_field"); fieldType.setHasDocValues(true ); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.DOUBLE) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.DOUBLE) .executionHint(executionHint) .includeExclude(new IncludeExclude(new double[]{0.0, 5.0}, null)) .field("double_field") @@ -509,7 +515,7 @@ public void testNumericIncludeExclude() throws Exception { assertEquals(1L, result.getBuckets().get(1).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue((InternalTerms)result)); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.DOUBLE) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.DOUBLE) .executionHint(executionHint) .includeExclude(new IncludeExclude(null, new double[]{0.0, 5.0})) .field("double_field") @@ -670,7 +676,8 @@ private void termsAggregator(ValueType valueType, MappedFieldType fieldType, String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString(); logger.info("bucket_order={} size={} execution_hint={}", bucketOrder, size, executionHint); IndexSearcher indexSearcher = newIndexSearcher(indexReader); - AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueType) + AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .userValueTypeHint(valueType) .executionHint(executionHint) .size(size) .shardSize(size) @@ -700,7 +707,8 @@ private void termsAggregator(ValueType valueType, MappedFieldType fieldType, MappedFieldType filterFieldType = new KeywordFieldMapper.KeywordFieldType(); filterFieldType.setName("include"); aggregationBuilder = new FilterAggregationBuilder("_name1", QueryBuilders.termQuery("include", "yes")); - aggregationBuilder.subAggregation(new TermsAggregationBuilder("_name2", valueType) + aggregationBuilder.subAggregation(new TermsAggregationBuilder("_name2") + .userValueTypeHint(valueType) .executionHint(executionHint) .size(numTerms) .collectMode(randomFrom(Aggregator.SubAggCollectionMode.values())) @@ -766,7 +774,8 @@ private void termsAggregatorWithNestedMaxAgg(ValueType valueType, MappedFiel logger.info("bucket_order={} size={} execution_hint={}, collect_mode={}", bucketOrder, size, executionHint, collectionMode); IndexSearcher indexSearcher = newIndexSearcher(indexReader); - AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueType) + AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .userValueTypeHint(valueType) .executionHint(executionHint) .collectMode(collectionMode) .size(size) @@ -812,7 +821,8 @@ public void testEmpty() throws Exception { fieldType1.setHasDocValues(true); try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { IndexSearcher indexSearcher = newIndexSearcher(indexReader); - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .userValueTypeHint(ValueType.STRING) .field("string"); Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1); aggregator.preCollection(); @@ -822,7 +832,7 @@ public void testEmpty() throws Exception { assertEquals("_name", result.getName()); assertEquals(0, result.getBuckets().size()); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.LONG) .field("long"); aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType2); aggregator.preCollection(); @@ -832,7 +842,7 @@ public void testEmpty() throws Exception { assertEquals("_name", result.getName()); assertEquals(0, result.getBuckets().size()); - aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.DOUBLE) + aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.DOUBLE) .field("double"); aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType3); aggregator.preCollection(); @@ -854,7 +864,8 @@ public void testUnmapped() throws Exception { ValueType[] valueTypes = new ValueType[]{ValueType.STRING, ValueType.LONG, ValueType.DOUBLE}; String[] fieldNames = new String[]{"string", "long", "double"}; for (int i = 0; i < fieldNames.length; i++) { - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueTypes[i]) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .userValueTypeHint(valueTypes[i]) .field(fieldNames[i]); Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, (MappedFieldType) null); aggregator.preCollection(); @@ -891,7 +902,8 @@ public void testUnmappedWithMissing() throws Exception { for (int i = 0; i < fieldNames.length; i++) { - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueTypes[i]) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .userValueTypeHint(valueTypes[i]) .field(fieldNames[i]).missing(missingValues[i]); Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1); aggregator.preCollection(); @@ -926,9 +938,8 @@ public void testRangeField() throws Exception { fieldType.setName(fieldName); IndexSearcher indexSearcher = newIndexSearcher(indexReader); - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", null) .field(fieldName); - // Note - other places we throw IllegalArgumentException - expectThrows(AggregationExecutionException.class, () -> { + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .field(fieldName); + expectThrows(IllegalArgumentException.class, () -> { createAggregator(aggregationBuilder, indexSearcher, fieldType); }); } @@ -950,9 +961,8 @@ public void testGeoPointField() throws Exception { fieldType.setName("field"); IndexSearcher indexSearcher = newIndexSearcher(indexReader); - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", null) .field(field); - // Note - other places we throw IllegalArgumentException - expectThrows(AggregationExecutionException.class, () -> { + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .field(field); + expectThrows(IllegalArgumentException.class, () -> { createAggregator(aggregationBuilder, indexSearcher, fieldType); }); } @@ -974,7 +984,7 @@ public void testIpField() throws Exception { fieldType.setName("field"); IndexSearcher indexSearcher = newIndexSearcher(indexReader); - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", null) .field(field); + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .field(field); // Note - other places we throw IllegalArgumentException Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -1009,12 +1019,13 @@ public void testNestedTermsAgg() throws Exception { IndexSearcher indexSearcher = newIndexSearcher(indexReader); String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString(); Aggregator.SubAggCollectionMode collectionMode = randomFrom(Aggregator.SubAggCollectionMode.values()); - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name1", ValueType.STRING) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name1") + .userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .collectMode(collectionMode) .field("field1") .order(BucketOrder.key(true)) - .subAggregation(new TermsAggregationBuilder("_name2", ValueType.STRING) + .subAggregation(new TermsAggregationBuilder("_name2").userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .collectMode(collectionMode) .field("field2") @@ -1052,7 +1063,7 @@ public void testNestedTermsAgg() throws Exception { public void testMixLongAndDouble() throws Exception { for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) { - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.LONG) .executionHint(executionMode.toString()) .field("number") .order(BucketOrder.key(true)); @@ -1124,13 +1135,13 @@ public void testGlobalAggregationWithScore() throws IOException { Aggregator.SubAggCollectionMode collectionMode = randomFrom(Aggregator.SubAggCollectionMode.values()); GlobalAggregationBuilder globalBuilder = new GlobalAggregationBuilder("global") .subAggregation( - new TermsAggregationBuilder("terms", ValueType.STRING) + new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .collectMode(collectionMode) .field("keyword") .order(BucketOrder.key(true)) .subAggregation( - new TermsAggregationBuilder("sub_terms", ValueType.STRING) + new TermsAggregationBuilder("sub_terms").userValueTypeHint(ValueType.STRING) .executionHint(executionHint) .collectMode(collectionMode) .field("keyword").order(BucketOrder.key(true)) @@ -1177,7 +1188,7 @@ public void testWithNestedAggregations() throws IOException { for (Aggregator.SubAggCollectionMode mode : Aggregator.SubAggCollectionMode.values()) { for (boolean withScore : new boolean[]{true, false}) { NestedAggregationBuilder nested = new NestedAggregationBuilder("nested", "nested_object") - .subAggregation(new TermsAggregationBuilder("terms", ValueType.LONG) + .subAggregation(new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.LONG) .field("nested_value") // force the breadth_first mode .collectMode(mode) @@ -1243,7 +1254,7 @@ public void testOrderByPipelineAggregation() throws Exception { "script", new Script("2.718")); TermsAggregationBuilder termsAgg = terms("terms") .field("field") - .valueType(ValueType.STRING) + .userValueTypeHint(ValueType.STRING) .order(BucketOrder.aggregation("script", true)) .subAggregation(bucketScriptAgg); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java index b34132a51e00a..2b4af02a7cddd 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java @@ -524,7 +524,7 @@ public void testOrderByEmptyAggregation() throws IOException { fieldType.setName("value"); fieldType.setHasDocValues(true); - AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("terms", ValueType.NUMERIC) + AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.NUMERIC) .field("value") .order(BucketOrder.compound(BucketOrder.aggregation("filter>avg", true))) .subAggregation(AggregationBuilders.filter("filter", termQuery("value", 100)) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java index 53a01d08bdcf2..48315fec2c51f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java @@ -64,7 +64,7 @@ public void testRangeFieldValues() throws IOException { final String fieldName = "rangeField"; MappedFieldType fieldType = new RangeFieldMapper.Builder(fieldName, rangeType).fieldType(); fieldType.setName(fieldName); - final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("_name", null).field(fieldName); + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("_name").field(fieldName); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { iw.addDocument(singleton(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(singleton(range1))))); iw.addDocument(singleton(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(singleton(range1))))); @@ -131,7 +131,7 @@ public void testQueryFiltersAll() throws IOException { } public void testUnmappedMissingString() throws IOException { - CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name", null) + CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name") .field("number").missing("🍌🍌🍌"); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { @@ -145,7 +145,7 @@ public void testUnmappedMissingString() throws IOException { } public void testUnmappedMissingNumber() throws IOException { - CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name", null) + CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name") .field("number").missing(1234); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { @@ -159,7 +159,7 @@ public void testUnmappedMissingNumber() throws IOException { } public void testUnmappedMissingGeoPoint() throws IOException { - CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name", null) + CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name") .field("number").missing(new GeoPoint(42.39561, -71.13051)); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { @@ -177,7 +177,7 @@ private void testCase(Query query, CheckedConsumer { @Override protected final CardinalityAggregationBuilder createTestAggregatorBuilder() { - CardinalityAggregationBuilder factory = new CardinalityAggregationBuilder(randomAlphaOfLengthBetween(3, 10), null); + CardinalityAggregationBuilder factory = new CardinalityAggregationBuilder(randomAlphaOfLengthBetween(3, 10)); String field = randomNumericField(); randomFieldOrScript(factory, field); if (randomBoolean()) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java index 83713ff52af84..001f18e8eea47 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java @@ -42,6 +42,8 @@ public class ExtendedStatsAggregatorTests extends AggregatorTestCase { private static final double TOLERANCE = 1e-5; + // TODO: Add script test cases. Should fail with defaultValuesSourceType() commented out. + public void testEmpty() throws IOException { MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java index 4f27cc1e3099e..c692d45371039 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java @@ -28,11 +28,15 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.test.geo.RandomGeoGenerator; import java.io.IOException; +import java.util.List; public class GeoCentroidAggregatorTests extends AggregatorTestCase { @@ -178,4 +182,13 @@ private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) thro } } + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new GeoCentroidAggregationBuilder("foo").field(fieldName); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of(CoreValuesSourceType.GEOPOINT); + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java index 7ed434f3a202d..6c34ba7430a38 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java @@ -53,7 +53,9 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.NUMERIC); + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN); } public void testEmpty() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorTests.java index c70cfc459fa96..a0a0c5a9118a0 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorTests.java @@ -19,9 +19,11 @@ package org.elasticsearch.search.aggregations.metrics; +import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; @@ -30,9 +32,13 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; @@ -40,6 +46,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.function.Consumer; @@ -59,7 +66,9 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.NUMERIC); + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN); } public void testNoDocs() throws IOException { @@ -71,6 +80,39 @@ public void testNoDocs() throws IOException { }); } + /** + * Attempting to use HDRPercentileAggregation on a string field throws IllegalArgumentException + */ + public void testStringField() throws IOException { + final String fieldName = "string"; + MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType(); + fieldType.setName(fieldName); + fieldType.setHasDocValues(true); + expectThrows(IllegalArgumentException.class, + () -> testCase(new DocValuesFieldExistsQuery(fieldName), iw -> { + iw.addDocument(singleton(new SortedSetDocValuesField("string", new BytesRef("bogus")))); + iw.addDocument(singleton(new SortedSetDocValuesField("string", new BytesRef("zwomp")))); + iw.addDocument(singleton(new SortedSetDocValuesField("string", new BytesRef("foobar")))); + }, hdr -> {}, fieldType, fieldName)); + } + + /** + * Attempting to use HDRPercentileAggregation on a range field throws IllegalArgumentException + */ + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/42949") + public void testRangeField() throws IOException { + // Currently fails (throws ClassCast exception), but should be fixed once HDRPercentileAggregation uses the ValuesSource registry + final String fieldName = "range"; + MappedFieldType fieldType = new RangeFieldMapper.Builder(fieldName, RangeType.DOUBLE).fieldType(); + fieldType.setName(fieldName); + RangeFieldMapper.Range range =new RangeFieldMapper.Range(RangeType.DOUBLE, 1.0D, 5.0D, true, true); + BytesRef encodedRange = RangeType.DOUBLE.encodeRanges(Collections.singleton(range)); + expectThrows(IllegalArgumentException.class, + () -> testCase(new DocValuesFieldExistsQuery(fieldName), iw -> { + iw.addDocument(singleton(new BinaryDocValuesField(fieldName, encodedRange))); + }, hdr -> {}, fieldType, fieldName)); + } + public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7))); @@ -149,6 +191,13 @@ public void testHdrThenTdigestSettings() throws Exception { private void testCase(Query query, CheckedConsumer buildIndex, Consumer verify) throws IOException { + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + fieldType.setName("number"); + testCase(query, buildIndex, verify, fieldType, "number"); + } + + private void testCase(Query query, CheckedConsumer buildIndex, + Consumer verify, MappedFieldType fieldType, String fieldName) throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { buildIndex.accept(indexWriter); @@ -160,14 +209,12 @@ private void testCase(Query query, CheckedConsumermax", true))) .subAggregation(AggregationBuilders.filter("filter", termQuery("value", 100)) @@ -831,7 +831,7 @@ public void testEarlyTermination() throws Exception { IndexSearcher indexSearcher = newSearcher(indexReader, true, true); MaxAggregationBuilder maxAggregationBuilder = new MaxAggregationBuilder("max") .field("values"); - ValueCountAggregationBuilder countAggregationBuilder = new ValueCountAggregationBuilder("count", null) + ValueCountAggregationBuilder countAggregationBuilder = new ValueCountAggregationBuilder("count") .field("values"); MaxAggregator maxAggregator = createAggregator(maxAggregationBuilder, indexSearcher, fieldType); @@ -881,9 +881,10 @@ public void testNestedEarlyTermination() throws Exception { for (Aggregator.SubAggCollectionMode collectionMode : Aggregator.SubAggCollectionMode.values()) { MaxAggregationBuilder maxAggregationBuilder = new MaxAggregationBuilder("max") .field("values"); - ValueCountAggregationBuilder countAggregationBuilder = new ValueCountAggregationBuilder("count", null) + ValueCountAggregationBuilder countAggregationBuilder = new ValueCountAggregationBuilder("count") .field("values"); - TermsAggregationBuilder termsAggregationBuilder = new TermsAggregationBuilder("terms", ValueType.NUMERIC) + TermsAggregationBuilder termsAggregationBuilder = new TermsAggregationBuilder("terms") + .userValueTypeHint(ValueType.NUMERIC) .field("value").collectMode(collectionMode) .subAggregation(new MaxAggregationBuilder("sub_max").field("invalid")); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index cc58333d379bc..914bf3a13de4f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -23,10 +23,12 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexOptions; @@ -44,6 +46,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.CheckedConsumer; @@ -53,6 +56,8 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; @@ -81,12 +86,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.FieldContext; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.LeafDocLookup; @@ -262,6 +263,20 @@ public void testQueryFiltersAll() throws IOException { }); } + public void testIpField() throws IOException { + final String fieldName = "IP_field"; + MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("min").field(fieldName); + + MappedFieldType fieldType = new IpFieldMapper.IpFieldType(); + fieldType.setName(fieldName); + + boolean v4 = randomBoolean(); + expectThrows(IllegalArgumentException.class, () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(randomIp(v4)))))); + iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(randomIp(v4)))))); + }, min -> fail("expected an exception"), fieldType)); + } + public void testUnmappedWithMissingField() throws IOException { MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("min").field("does_not_exist").missing(0L); @@ -277,6 +292,22 @@ public void testUnmappedWithMissingField() throws IOException { }, fieldType); } + public void testUnsupportedType() { + MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("min").field("not_a_number"); + + MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType(); + fieldType.setName("not_a_number"); + fieldType.setHasDocValues(true); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(singleton(new SortedSetDocValuesField("string", new BytesRef("foo")))); + }, (Consumer) min -> { + fail("Should have thrown exception"); + }, fieldType)); + assertEquals("Field [not_a_number] of type [keyword(indexed,tokenized)] is not supported for aggregation [min]", e.getMessage()); + } + public void testBadMissingField() { MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("min").field("number").missing("not_a_number"); @@ -583,7 +614,7 @@ public void testMultiValuedFieldWithScriptParams() throws IOException { } public void testOrderByEmptyAggregation() throws IOException { - AggregationBuilder termsBuilder = new TermsAggregationBuilder("terms", ValueType.NUMERIC) + AggregationBuilder termsBuilder = new TermsAggregationBuilder("terms") .field("number") .order(BucketOrder.compound(BucketOrder.aggregation("filter>min", true))) .subAggregation(new FilterAggregationBuilder("filter", termQuery("number", 100)) @@ -649,11 +680,11 @@ public void testScriptCaching() throws IOException { fieldType.setName("number"); MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("min") .field("number") - .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, INVERT_SCRIPT, Collections.emptyMap()));; + .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, INVERT_SCRIPT, Collections.emptyMap())); MinAggregationBuilder nonDeterministicAggregationBuilder = new MinAggregationBuilder("min") .field("number") - .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, RANDOM_SCRIPT, Collections.emptyMap()));; + .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, RANDOM_SCRIPT, Collections.emptyMap())); try (Directory directory = newDirectory()) { RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); @@ -865,10 +896,10 @@ private Aggregator mockAggregator() { return mock(Aggregator.class); } - private ValuesSourceConfig mockNumericValuesSourceConfig(String fieldName, - NumberFieldMapper.NumberType numType, - boolean indexed) { - ValuesSourceConfig config = mock(ValuesSourceConfig.class); + private ValuesSourceConfig mockNumericValuesSourceConfig(String fieldName, + NumberFieldMapper.NumberType numType, + boolean indexed) { + ValuesSourceConfig config = mock(ValuesSourceConfig.class); MappedFieldType ft = new NumberFieldMapper.NumberFieldType(numType); ft.setName(fieldName); ft.setIndexOptions(indexed ? IndexOptions.DOCS : IndexOptions.NONE); @@ -877,9 +908,9 @@ private ValuesSourceConfig mockNumericValuesSourceConfig(S return config; } - private ValuesSourceConfig mockDateValuesSourceConfig(String fieldName, boolean indexed, + private ValuesSourceConfig mockDateValuesSourceConfig(String fieldName, boolean indexed, DateFieldMapper.Resolution resolution) { - ValuesSourceConfig config = mock(ValuesSourceConfig.class); + ValuesSourceConfig config = mock(ValuesSourceConfig.class); Mapper.BuilderContext builderContext = new Mapper.BuilderContext( Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(), new ContentPath()); @@ -919,13 +950,5 @@ private void testC } } - @Override - protected List getSupportedValuesSourceTypes() { - return Collections.singletonList(CoreValuesSourceType.NUMERIC); - } - @Override - protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { - return new MinAggregationBuilder("foo").field(fieldName); - } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index 44ca8820fdfd8..56206f268e1c0 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -426,7 +426,8 @@ protected QueryShardContext queryShardContextMock(IndexSearcher searcher, MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); - return new QueryShardContext(0, indexSettings, bigArrays, null, null, mapperService, null, scriptService, - xContentRegistry(), writableRegistry(), null, null, System::currentTimeMillis, null, null, () -> true); + return new QueryShardContext(0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, + null, mapperService, null, scriptService, xContentRegistry(), writableRegistry(), + null, null, System::currentTimeMillis, null, null, () -> true, null); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java index ccd01c94de2f3..16fd9211a9ee4 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java @@ -72,6 +72,8 @@ public class StatsAggregatorTests extends AggregatorTestCase { private static final String VALUE_SCRIPT_NAME = "value_script"; private static final String FIELD_SCRIPT_NAME = "field_script"; + // TODO: Script tests, should fail with defaultValuesSourceType disabled. + public void testEmpty() throws IOException { final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG); ft.setName("field"); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java index 420c9b8cecbcf..043554bcc4dc3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java @@ -53,7 +53,9 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.NUMERIC); + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN); } public void testEmpty() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorTests.java index ea2aca16aa8a2..63f8e799fb796 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorTests.java @@ -59,7 +59,9 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.NUMERIC); + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN); } public void testNoDocs() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java index 559aec4d231dc..094a4b093cf34 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java @@ -82,7 +82,7 @@ public class ValueCountAggregatorTests extends AggregatorTestCase { @Override protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { - return new ValueCountAggregationBuilder("foo", null).field(fieldName); + return new ValueCountAggregationBuilder("foo").field(fieldName); } @Override @@ -91,8 +91,7 @@ protected List getSupportedValuesSourceTypes() { CoreValuesSourceType.NUMERIC, CoreValuesSourceType.BYTES, CoreValuesSourceType.GEOPOINT, - CoreValuesSourceType.RANGE, - CoreValuesSourceType.HISTOGRAM + CoreValuesSourceType.RANGE ); } @@ -177,7 +176,7 @@ public void testQueryFiltersAll() throws IOException { } public void testUnmappedMissingString() throws IOException { - ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null) + ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name") .field("number").missing("🍌🍌🍌"); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { @@ -191,7 +190,7 @@ public void testUnmappedMissingString() throws IOException { } public void testUnmappedMissingNumber() throws IOException { - ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null) + ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name") .field("number").missing(1234); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { @@ -205,7 +204,7 @@ public void testUnmappedMissingNumber() throws IOException { } public void testUnmappedMissingGeoPoint() throws IOException { - ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null) + ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name") .field("number").missing(new GeoPoint(42.39561, -71.13051)); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { @@ -225,7 +224,7 @@ public void testRangeFieldValues() throws IOException { final String fieldName = "rangeField"; MappedFieldType fieldType = new RangeFieldMapper.Builder(fieldName, rangeType).fieldType(); fieldType.setName(fieldName); - final ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("_name", null).field(fieldName); + final ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("_name").field(fieldName); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { iw.addDocument(singleton(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(singleton(range1))))); iw.addDocument(singleton(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(singleton(range1))))); @@ -238,7 +237,7 @@ public void testRangeFieldValues() throws IOException { } public void testValueScriptNumber() throws IOException { - ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null) + ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name") .field(FIELD_NAME) .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap())); @@ -257,7 +256,7 @@ public void testValueScriptNumber() throws IOException { } public void testSingleScriptNumber() throws IOException { - ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null) + ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name") .field(FIELD_NAME); MappedFieldType fieldType = createMappedFieldType(ValueType.NUMERIC); @@ -287,7 +286,7 @@ public void testSingleScriptNumber() throws IOException { } public void testValueScriptString() throws IOException { - ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null) + ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name") .field(FIELD_NAME) .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap())); @@ -306,7 +305,7 @@ public void testValueScriptString() throws IOException { } public void testSingleScriptString() throws IOException { - ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null) + ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name") .field(FIELD_NAME); MappedFieldType fieldType = createMappedFieldType(ValueType.STRING); @@ -340,11 +339,23 @@ private void testCase(Query query, ValueType valueType, CheckedConsumer indexer, Consumer verify) throws IOException { + // Test both with and without the userValueTypeHint + testCase(query, valueType, indexer, verify, true); + testCase(query, valueType, indexer, verify, false); + } + + private void testCase(Query query, + ValueType valueType, + CheckedConsumer indexer, + Consumer verify, boolean testWithHint) throws IOException { MappedFieldType fieldType = createMappedFieldType(valueType); fieldType.setName(FIELD_NAME); fieldType.setHasDocValues(true); - ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("_name", valueType); + ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("_name"); + if (valueType != null && testWithHint) { + aggregationBuilder.userValueTypeHint(valueType); + } aggregationBuilder.field(FIELD_NAME); testCase(aggregationBuilder, query, indexer, verify, fieldType); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java index 591c096eea147..846bf79afc8d2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java @@ -25,7 +25,7 @@ public class ValueCountTests extends BaseAggregationTestCase aggBuilders = new HashSet<>(); aggBuilders.add(singleBucketAgg); aggBuilders.add(multiBucketAgg); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java index 97e5fac24dcda..8f2eaa83509ce 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java @@ -78,7 +78,7 @@ public void testScript() throws IOException { fieldType1.setHasDocValues(true); FiltersAggregationBuilder filters = new FiltersAggregationBuilder("placeholder", new MatchAllQueryBuilder()) - .subAggregation(new TermsAggregationBuilder("the_terms", ValueType.STRING).field("the_field") + .subAggregation(new TermsAggregationBuilder("the_terms").userValueTypeHint(ValueType.STRING).field("the_field") .subAggregation(new AvgAggregationBuilder("the_avg").field("number_field"))) .subAggregation(new BucketScriptPipelineAggregationBuilder("bucket_script", Collections.singletonMap("the_avg", "the_terms['test1']>the_avg.value"), diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java index 90cc9be95e3be..a7e2db4bada33 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java @@ -62,7 +62,7 @@ public void testSigmaFromInt() throws Exception { public void testValidate() { AggregationBuilder singleBucketAgg = new GlobalAggregationBuilder("global"); - AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms", ValueType.STRING); + AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING); final Set aggBuilders = new HashSet<>(); aggBuilders.add(singleBucketAgg); aggBuilders.add(multiBucketAgg); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java index edbc1cda3eae2..7713a9f8d095c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java @@ -39,7 +39,7 @@ protected MaxBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(Stri public void testValidate() { AggregationBuilder singleBucketAgg = new GlobalAggregationBuilder("global"); - AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms", ValueType.STRING); + AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING); final Set aggBuilders = new HashSet<>(); aggBuilders.add(singleBucketAgg); aggBuilders.add(multiBucketAgg); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java index 057e074a90cfc..e9d3e15a3fa2b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java @@ -39,7 +39,7 @@ protected MinBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(Stri public void testValidate() { AggregationBuilder singleBucketAgg = new GlobalAggregationBuilder("global"); - AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms", ValueType.STRING); + AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING); final Set aggBuilders = new HashSet<>(); aggBuilders.add(singleBucketAgg); aggBuilders.add(multiBucketAgg); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java index 1918d26e1e1af..0b8757441cdb4 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java @@ -66,7 +66,7 @@ public void testPercentsFromMixedArray() throws Exception { public void testValidate() { AggregationBuilder singleBucketAgg = new GlobalAggregationBuilder("global"); - AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms", ValueType.STRING); + AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING); final Set aggBuilders = new HashSet<>(); aggBuilders.add(singleBucketAgg); aggBuilders.add(multiBucketAgg); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java index 2d67735468984..02e932d607d2a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java @@ -115,7 +115,7 @@ public static class MockBucket { * @param values Array of values to compute metric for * @param metric A metric builder which defines what kind of metric should be returned for the values */ - public static double calculateMetric(double[] values, ValuesSourceAggregationBuilder metric) { + public static double calculateMetric(double[] values, ValuesSourceAggregationBuilder metric) { if (metric instanceof MinAggregationBuilder) { double accumulator = Double.POSITIVE_INFINITY; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffIT.java index 33c3de9c1f46d..f36aafad7d32f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.collect.EvictingQueue; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; @@ -57,7 +56,7 @@ public class SerialDiffIT extends ESIntegTestCase { static int numBuckets; static int lag; static BucketHelpers.GapPolicy gapPolicy; - static ValuesSourceAggregationBuilder> metric; + static ValuesSourceAggregationBuilder> metric; static List mockHisto; static Map> testValues; @@ -77,8 +76,8 @@ public String toString(){ } } - private ValuesSourceAggregationBuilder> randomMetric(String name, String field) { + private ValuesSourceAggregationBuilder< + ? extends ValuesSourceAggregationBuilder> randomMetric(String name, String field) { int rand = randomIntBetween(0,3); switch (rand) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java index aac81c19be6bd..dffef51f643dd 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java @@ -40,7 +40,7 @@ protected StatsBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(St public void testValidate() { AggregationBuilder singleBucketAgg = new GlobalAggregationBuilder("global"); - AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms", ValueType.STRING); + AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING); final Set aggBuilders = new HashSet<>(); aggBuilders.add(singleBucketAgg); aggBuilders.add(multiBucketAgg); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java index 0bcbf592458aa..fc8c1be801313 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java @@ -39,7 +39,7 @@ protected SumBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(Stri public void testValidate() { AggregationBuilder singleBucketAgg = new GlobalAggregationBuilder("global"); - AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms", ValueType.STRING); + AggregationBuilder multiBucketAgg = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING); final Set aggBuilders = new HashSet<>(); aggBuilders.add(singleBucketAgg); aggBuilders.add(multiBucketAgg); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java index 5f57a8ee9c12e..a9f500bd09154 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java @@ -19,60 +19,21 @@ package org.elasticsearch.search.aggregations.support; -import org.elasticsearch.common.io.stream.AbstractWriteableEnumTestCase; - -import java.io.IOException; +import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.equalTo; -public class CoreValuesSourceTypeTests extends AbstractWriteableEnumTestCase { - - public CoreValuesSourceTypeTests() { - super(CoreValuesSourceType::fromStream); - } - - @Override - public void testValidOrdinals() { - assertThat(CoreValuesSourceType.ANY.ordinal(), equalTo(0)); - assertThat(CoreValuesSourceType.NUMERIC.ordinal(), equalTo(1)); - assertThat(CoreValuesSourceType.BYTES.ordinal(), equalTo(2)); - assertThat(CoreValuesSourceType.GEOPOINT.ordinal(), equalTo(3)); - assertThat(CoreValuesSourceType.RANGE.ordinal(), equalTo(4)); - assertThat(CoreValuesSourceType.HISTOGRAM.ordinal(), equalTo(5)); - } +public class CoreValuesSourceTypeTests extends ESTestCase { - @Override public void testFromString() { - assertThat(CoreValuesSourceType.fromString("any"), equalTo(CoreValuesSourceType.ANY)); assertThat(CoreValuesSourceType.fromString("numeric"), equalTo(CoreValuesSourceType.NUMERIC)); assertThat(CoreValuesSourceType.fromString("bytes"), equalTo(CoreValuesSourceType.BYTES)); assertThat(CoreValuesSourceType.fromString("geopoint"), equalTo(CoreValuesSourceType.GEOPOINT)); assertThat(CoreValuesSourceType.fromString("range"), equalTo(CoreValuesSourceType.RANGE)); - assertThat(CoreValuesSourceType.fromString("histogram"), equalTo(CoreValuesSourceType.HISTOGRAM)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> CoreValuesSourceType.fromString("does_not_exist")); assertThat(e.getMessage(), equalTo("No enum constant org.elasticsearch.search.aggregations.support.CoreValuesSourceType.DOES_NOT_EXIST")); expectThrows(NullPointerException.class, () -> CoreValuesSourceType.fromString(null)); } - - @Override - public void testReadFrom() throws IOException { - assertReadFromStream(0, CoreValuesSourceType.ANY); - assertReadFromStream(1, CoreValuesSourceType.NUMERIC); - assertReadFromStream(2, CoreValuesSourceType.BYTES); - assertReadFromStream(3, CoreValuesSourceType.GEOPOINT); - assertReadFromStream(4, CoreValuesSourceType.RANGE); - assertReadFromStream(5, CoreValuesSourceType.HISTOGRAM); - } - - @Override - public void testWriteTo() throws IOException { - assertWriteToStream(CoreValuesSourceType.ANY, 0); - assertWriteToStream(CoreValuesSourceType.NUMERIC, 1); - assertWriteToStream(CoreValuesSourceType.BYTES, 2); - assertWriteToStream(CoreValuesSourceType.GEOPOINT, 3); - assertWriteToStream(CoreValuesSourceType.RANGE, 4); - assertWriteToStream(CoreValuesSourceType.HISTOGRAM, 5); - } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValueTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValueTypeTests.java index 84995277fb7da..9bc7eb9a22988 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValueTypeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValueTypeTests.java @@ -24,26 +24,59 @@ public class ValueTypeTests extends ESTestCase { public void testResolve() { - assertEquals(ValueType.STRING, ValueType.resolveForScript("string")); - assertEquals(ValueType.DOUBLE, ValueType.resolveForScript("float")); - assertEquals(ValueType.DOUBLE, ValueType.resolveForScript("double")); - assertEquals(ValueType.LONG, ValueType.resolveForScript("byte")); - assertEquals(ValueType.LONG, ValueType.resolveForScript("short")); - assertEquals(ValueType.LONG, ValueType.resolveForScript("integer")); - assertEquals(ValueType.LONG, ValueType.resolveForScript("long")); - assertEquals(ValueType.DATE, ValueType.resolveForScript("date")); - assertEquals(ValueType.IP, ValueType.resolveForScript("ip")); - assertEquals(ValueType.BOOLEAN, ValueType.resolveForScript("boolean")); + assertEquals(ValueType.STRING, ValueType.lenientParse("string")); + assertEquals(ValueType.DOUBLE, ValueType.lenientParse("float")); + assertEquals(ValueType.DOUBLE, ValueType.lenientParse("double")); + assertEquals(ValueType.LONG, ValueType.lenientParse("byte")); + assertEquals(ValueType.LONG, ValueType.lenientParse("short")); + assertEquals(ValueType.LONG, ValueType.lenientParse("integer")); + assertEquals(ValueType.LONG, ValueType.lenientParse("long")); + assertEquals(ValueType.DATE, ValueType.lenientParse("date")); + assertEquals(ValueType.IP, ValueType.lenientParse("ip")); + assertEquals(ValueType.BOOLEAN, ValueType.lenientParse("boolean")); } public void testCompatibility() { assertTrue(ValueType.DOUBLE.isA(ValueType.NUMERIC)); + assertTrue(ValueType.DOUBLE.isA(ValueType.NUMBER)); + assertTrue(ValueType.DOUBLE.isA(ValueType.LONG)); + assertTrue(ValueType.DOUBLE.isA(ValueType.BOOLEAN)); + assertTrue(ValueType.DOUBLE.isA(ValueType.DATE)); + assertTrue(ValueType.DOUBLE.isA(ValueType.DOUBLE)); + + assertTrue(ValueType.LONG.isA(ValueType.NUMERIC)); + assertTrue(ValueType.LONG.isA(ValueType.NUMBER)); + assertTrue(ValueType.LONG.isA(ValueType.LONG)); + assertTrue(ValueType.LONG.isA(ValueType.BOOLEAN)); + assertTrue(ValueType.LONG.isA(ValueType.DATE)); + assertTrue(ValueType.LONG.isA(ValueType.DOUBLE)); + + assertTrue(ValueType.DATE.isA(ValueType.NUMERIC)); + assertTrue(ValueType.DATE.isA(ValueType.NUMBER)); assertTrue(ValueType.DATE.isA(ValueType.LONG)); + assertTrue(ValueType.DATE.isA(ValueType.BOOLEAN)); + assertTrue(ValueType.DATE.isA(ValueType.DATE)); + assertTrue(ValueType.DATE.isA(ValueType.DOUBLE)); + + assertTrue(ValueType.NUMERIC.isA(ValueType.NUMERIC)); assertTrue(ValueType.NUMERIC.isA(ValueType.NUMBER)); + assertTrue(ValueType.NUMERIC.isA(ValueType.LONG)); + assertTrue(ValueType.NUMERIC.isA(ValueType.BOOLEAN)); + assertTrue(ValueType.NUMERIC.isA(ValueType.DATE)); + assertTrue(ValueType.NUMERIC.isA(ValueType.DOUBLE)); + + assertTrue(ValueType.BOOLEAN.isA(ValueType.NUMERIC)); assertTrue(ValueType.BOOLEAN.isA(ValueType.NUMBER)); + assertTrue(ValueType.BOOLEAN.isA(ValueType.LONG)); + assertTrue(ValueType.BOOLEAN.isA(ValueType.BOOLEAN)); + assertTrue(ValueType.BOOLEAN.isA(ValueType.DATE)); + assertTrue(ValueType.BOOLEAN.isA(ValueType.DOUBLE)); + assertFalse(ValueType.STRING.isA(ValueType.NUMBER)); assertFalse(ValueType.DATE.isA(ValueType.IP)); - } + assertTrue(ValueType.IP.isA(ValueType.STRING)); + assertTrue(ValueType.STRING.isA(ValueType.IP)); + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java index db208044df1a4..25a71bd2ed39e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.test.ESSingleNodeTestCase; +// TODO: This whole set of tests needs to be rethought. public class ValuesSourceConfigTests extends ESSingleNodeTestCase { public void testKeyword() throws Exception { @@ -44,9 +45,9 @@ public void testKeyword() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, null, "bytes", null, null, null, null); - ValuesSource.Bytes valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "bytes", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Bytes valuesSource = (ValuesSource.Bytes) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); assertTrue(values.advanceExact(0)); @@ -66,16 +67,16 @@ public void testEmptyKeyword() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, null, "bytes", null, null, null, null); - ValuesSource.Bytes valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "bytes", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Bytes valuesSource = (ValuesSource.Bytes) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); assertFalse(values.advanceExact(0)); config = ValuesSourceConfig.resolve( - context, null, "bytes", null, "abc", null, null); - valuesSource = config.toValuesSource(context); + context, null, "bytes", null, "abc", null, null, CoreValuesSourceType.BYTES, null); + valuesSource = (ValuesSource.Bytes) config.toValuesSource(); values = valuesSource.bytesValues(ctx); assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); @@ -92,14 +93,14 @@ public void testUnmappedKeyword() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, ValueType.STRING, "bytes", null, null, null, null); - ValuesSource.Bytes valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, ValueType.STRING, "bytes", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Bytes valuesSource = (ValuesSource.Bytes) config.toValuesSource(); assertNull(valuesSource); config = ValuesSourceConfig.resolve( - context, ValueType.STRING, "bytes", null, "abc", null, null); - valuesSource = config.toValuesSource(context); + context, ValueType.STRING, "bytes", null, "abc", null, null, CoreValuesSourceType.BYTES, null); + valuesSource = (ValuesSource.Bytes) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); assertTrue(values.advanceExact(0)); @@ -119,9 +120,9 @@ public void testLong() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, null, "long", null, null, null, null); - ValuesSource.Numeric valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "long", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); assertTrue(values.advanceExact(0)); @@ -141,16 +142,16 @@ public void testEmptyLong() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, null, "long", null, null, null, null); - ValuesSource.Numeric valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "long", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); assertFalse(values.advanceExact(0)); config = ValuesSourceConfig.resolve( - context, null, "long", null, 42, null, null); - valuesSource = config.toValuesSource(context); + context, null, "long", null, 42, null, null, CoreValuesSourceType.BYTES, null); + valuesSource = (ValuesSource.Numeric) config.toValuesSource(); values = valuesSource.longValues(ctx); assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); @@ -168,14 +169,14 @@ public void testUnmappedLong() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, ValueType.NUMBER, "long", null, null, null, null); - ValuesSource.Numeric valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, ValueType.NUMBER, "long", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.toValuesSource(); assertNull(valuesSource); config = ValuesSourceConfig.resolve( - context, ValueType.NUMBER, "long", null, 42, null, null); - valuesSource = config.toValuesSource(context); + context, ValueType.NUMBER, "long", null, 42, null, null, CoreValuesSourceType.BYTES, null); + valuesSource = (ValuesSource.Numeric) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); assertTrue(values.advanceExact(0)); @@ -195,9 +196,9 @@ public void testBoolean() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, null, "bool", null, null, null, null); - ValuesSource.Numeric valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "bool", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); assertTrue(values.advanceExact(0)); @@ -217,16 +218,16 @@ public void testEmptyBoolean() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, null, "bool", null, null, null, null); - ValuesSource.Numeric valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "bool", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); assertFalse(values.advanceExact(0)); config = ValuesSourceConfig.resolve( - context, null, "bool", null, true, null, null); - valuesSource = config.toValuesSource(context); + context, null, "bool", null, true, null, null, CoreValuesSourceType.BYTES, null); + valuesSource = (ValuesSource.Numeric) config.toValuesSource(); values = valuesSource.longValues(ctx); assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); @@ -244,14 +245,14 @@ public void testUnmappedBoolean() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, ValueType.BOOLEAN, "bool", null, null, null, null); - ValuesSource.Numeric valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, ValueType.BOOLEAN, "bool", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.toValuesSource(); assertNull(valuesSource); config = ValuesSourceConfig.resolve( - context, ValueType.BOOLEAN, "bool", null, true, null, null); - valuesSource = config.toValuesSource(context); + context, ValueType.BOOLEAN, "bool", null, true, null, null, CoreValuesSourceType.BYTES, null); + valuesSource = (ValuesSource.Numeric) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); assertTrue(values.advanceExact(0)); @@ -265,8 +266,8 @@ public void testTypeFieldDeprecation() { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, null, TypeFieldMapper.NAME, null, null, null, null); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, TypeFieldMapper.NAME, null, null, null, null, CoreValuesSourceType.BYTES, null); assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE); } } @@ -281,9 +282,9 @@ public void testFieldAlias() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); - ValuesSourceConfig config = ValuesSourceConfig.resolve( - context, ValueType.STRING, "alias", null, null, null, null); - ValuesSource.Bytes valuesSource = config.toValuesSource(context); + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, ValueType.STRING, "alias", null, null, null, null, CoreValuesSourceType.BYTES, null); + ValuesSource.Bytes valuesSource = (ValuesSource.Bytes) config.toValuesSource(); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index c1bef65f18cd1..aa470027aa300 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -280,7 +280,7 @@ public void testBuildSearchContextHighlight() throws IOException { // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, - null, null, System::currentTimeMillis, null, null, () -> true) { + null, null, System::currentTimeMillis, null, null, () -> true, null) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 341202d44a940..51f4ed5fc6ed4 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -144,7 +144,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true) { + xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); @@ -188,7 +188,7 @@ public void testRewritingKeepsSettings() throws IOException { // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true) { + xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index eda4871e32362..c6b0b66fd125e 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -200,7 +200,7 @@ protected final QueryShardContext createMockShardContext(IndexSearcher searcher) }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, - () -> randomNonNegativeLong(), null, null, () -> true) { + () -> randomNonNegativeLong(), null, null, () -> true, null) { @Override public MappedFieldType fieldMapper(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index 9bbd291b32a0d..7484b4fc54e21 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -181,7 +181,7 @@ public void testBuild() throws IOException { ((Script) invocation.getArguments()[0]).getIdOrCode())); QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, - System::currentTimeMillis, null, null, () -> true); + System::currentTimeMillis, null, null, () -> true, null); SuggestionContext suggestionContext = suggestionBuilder.build(mockShardContext); assertEquals(toBytesRef(suggestionBuilder.text()), suggestionContext.getText()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 2139268fe3888..3fe454de04ffb 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1272,7 +1272,8 @@ public void onFailure(final Exception e) { client, new MetaStateService(nodeEnv, namedXContentRegistry), Collections.emptyList(), - emptyMap() + emptyMap(), + null ); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); final ActionFilters actionFilters = new ActionFilters(emptySet()); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 31a6eac89ef37..2dd4e3890faaf 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -71,15 +71,12 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.BinaryFieldMapper; -import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldAliasMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; -import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.Mapper.BuilderContext; @@ -100,10 +97,12 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.subphase.FetchDocValuesPhase; @@ -114,6 +113,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.After; +import org.junit.BeforeClass; import java.io.IOException; import java.net.InetAddress; @@ -144,6 +144,7 @@ public abstract class AggregatorTestCase extends ESTestCase { private static final String NESTEDFIELD_PREFIX = "nested_"; private List releasables = new ArrayList<>(); private static final String TYPE_NAME = "type"; + protected static ValuesSourceRegistry valuesSourceRegistry; // A list of field types that should not be tested, or are not currently supported private static List TYPE_TEST_BLACKLIST = List.of( @@ -173,8 +174,12 @@ private static void registerFieldTypes(SearchContext searchContext, MapperServic when(mapperService.fieldType(fieldName)).thenReturn(fieldType); when(searchContext.smartNameFieldType(fieldName)).thenReturn(fieldType); } + } - + @BeforeClass + public static void initValuesSourceRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of()); + valuesSourceRegistry = searchModule.getValuesSourceRegistry(); } protected A createAggregator(AggregationBuilder aggregationBuilder, @@ -350,7 +355,8 @@ protected QueryShardContext queryShardContextMock(IndexSearcher searcher, return new QueryShardContext(0, indexSettings, bigArrays, null, getIndexFieldDataLookup(mapperService, circuitBreakerService), mapperService, null, getMockScriptService(), xContentRegistry(), - writableRegistry(), null, searcher, System::currentTimeMillis, null, null, () -> true); + writableRegistry(), null, searcher, System::currentTimeMillis, null, null, () -> true, + valuesSourceRegistry); } /** @@ -636,9 +642,6 @@ public final void testSupportedFieldTypes() throws IOException { if (supportedVSTypes.isEmpty()) { // If the test says it doesn't support any VStypes, it has not been converted yet so skip return; - } else if (supportedVSTypes.contains(CoreValuesSourceType.ANY)) { - throw new IllegalArgumentException("Tests should not specify CoreValuesSourceType.ANY as a supported ValuesSourceType, " + - "but should instead list the concrete ValuesSourceTypes that are supported"); } for (Map.Entry mappedType : mapperRegistry.getMapperParsers().entrySet()) { @@ -708,17 +711,10 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd Document doc = new Document(); String json; + if (vst.equals(CoreValuesSourceType.NUMERIC)) { - // TODO note: once VS refactor adds DATE/BOOLEAN, this conditional will go away long v; - if (typeName.equals(DateFieldMapper.CONTENT_TYPE) || typeName.equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) { - // positive integer because date_nanos gets unhappy with large longs - v = Math.abs(randomInt()); - json = "{ \"" + fieldName + "\" : \"" + v + "\" }"; - } else if (typeName.equals(BooleanFieldMapper.CONTENT_TYPE)) { - v = randomBoolean() ? 0 : 1; - json = "{ \"" + fieldName + "\" : \"" + (v == 0 ? "false" : "true") + "\" }"; - } else if (typeName.equals(NumberFieldMapper.NumberType.DOUBLE.typeName())) { + if (typeName.equals(NumberFieldMapper.NumberType.DOUBLE.typeName())) { double d = Math.abs(randomDouble()); v = NumericUtils.doubleToSortableLong(d); json = "{ \"" + fieldName + "\" : \"" + d + "\" }"; @@ -741,15 +737,25 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd if (typeName.equals(BinaryFieldMapper.CONTENT_TYPE)) { doc.add(new BinaryFieldMapper.CustomBinaryDocValuesField(fieldName, new BytesRef("a").bytes)); json = "{ \"" + fieldName + "\" : \"a\" }"; - } else if (typeName.equals(IpFieldMapper.CONTENT_TYPE)) { - // TODO note: once VS refactor adds IP, this conditional will go away - InetAddress ip = randomIp(randomBoolean()); - json = "{ \"" + fieldName + "\" : \"" + NetworkAddress.format(ip) + "\" }"; - doc.add(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(ip)))); } else { doc.add(new SortedSetDocValuesField(fieldName, new BytesRef("a"))); json = "{ \"" + fieldName + "\" : \"a\" }"; } + } else if (vst.equals(CoreValuesSourceType.DATE)) { + // positive integer because date_nanos gets unhappy with large longs + long v; + v = Math.abs(randomInt()); + doc.add(new SortedNumericDocValuesField(fieldName, v)); + json = "{ \"" + fieldName + "\" : \"" + v + "\" }"; + } else if (vst.equals(CoreValuesSourceType.BOOLEAN)) { + long v; + v = randomBoolean() ? 0 : 1; + doc.add(new SortedNumericDocValuesField(fieldName, v)); + json = "{ \"" + fieldName + "\" : \"" + (v == 0 ? "false" : "true") + "\" }"; + } else if (vst.equals(CoreValuesSourceType.IP)) { + InetAddress ip = randomIp(randomBoolean()); + json = "{ \"" + fieldName + "\" : \"" + NetworkAddress.format(ip) + "\" }"; + doc.add(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(ip)))); } else if (vst.equals(CoreValuesSourceType.RANGE)) { Object start; Object end; diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java index 66188e57f4337..5c090caa8a14f 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java @@ -199,7 +199,7 @@ public String randomNumericField() { } } - protected void randomFieldOrScript(ValuesSourceAggregationBuilder factory, String field) { + protected void randomFieldOrScript(ValuesSourceAggregationBuilder factory, String field) { int choice = randomInt(2); switch (choice) { case 0: diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index a424501a15296..94ceb168d18b6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -414,7 +414,7 @@ public void close() throws IOException { QueryShardContext createShardContext(IndexSearcher searcher) { return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataService::getForField, mapperService, similarityService, scriptService, xContentRegistry, - namedWriteableRegistry, this.client, searcher, () -> nowInMillis, null, indexNameMatcher(), () -> true); + namedWriteableRegistry, this.client, searcher, () -> nowInMillis, null, indexNameMatcher(), () -> true, null); } ScriptModule createScriptModule(List scriptPlugins) { diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java index 684210f13c57f..cb32f0ae82087 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java @@ -43,7 +43,7 @@ public void testAssertNoInFlightContext() { final long nowInMillis = randomNonNegativeLong(); SearchContext s = new TestSearchContext(new QueryShardContext(0, new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true)) { + xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null)) { @Override public SearchShardTarget shardTarget() { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java index 55f8fbe056c53..295cb91ab0bd9 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java @@ -24,11 +24,13 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.analytics.action.AnalyticsInfoTransportAction; import org.elasticsearch.xpack.analytics.action.AnalyticsUsageTransportAction; import org.elasticsearch.xpack.analytics.action.TransportAnalyticsStatsAction; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; import org.elasticsearch.xpack.analytics.boxplot.BoxplotAggregationBuilder; import org.elasticsearch.xpack.analytics.boxplot.InternalBoxplot; import org.elasticsearch.xpack.analytics.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder; @@ -50,6 +52,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import static java.util.Collections.singletonList; @@ -79,12 +82,14 @@ public List getAggregations() { StringStatsAggregationBuilder.NAME, StringStatsAggregationBuilder::new, usage.track(AnalyticsUsage.Item.STRING_STATS, checkLicense(StringStatsAggregationBuilder.PARSER))) - .addResultReader(InternalStringStats::new), + .addResultReader(InternalStringStats::new) + .setAggregatorRegistrar(StringStatsAggregationBuilder::registerAggregators), new AggregationSpec( BoxplotAggregationBuilder.NAME, BoxplotAggregationBuilder::new, usage.track(AnalyticsUsage.Item.BOXPLOT, checkLicense(BoxplotAggregationBuilder.PARSER))) - .addResultReader(InternalBoxplot::new), + .addResultReader(InternalBoxplot::new) + .setAggregatorRegistrar(BoxplotAggregationBuilder::registerAggregators), new AggregationSpec( TopMetricsAggregationBuilder.NAME, TopMetricsAggregationBuilder::new, @@ -111,6 +116,12 @@ public Map getMappers() { return Collections.singletonMap(HistogramFieldMapper.CONTENT_TYPE, new HistogramFieldMapper.TypeParser()); } + @Override + public List> getBareAggregatorRegistrar() { + return List.of(AnalyticsPercentilesAggregatorFactory::registerPercentilesAggregator, + AnalyticsPercentilesAggregatorFactory::registerPercentileRanksAggregator); + } + @Override public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedHDRPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedHDRPercentilesAggregator.java new file mode 100644 index 0000000000000..86918db07656c --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedHDRPercentilesAggregator.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.HdrHistogram.DoubleHistogram; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ScoreMode; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.util.ArrayUtils; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ObjectArray; +import org.elasticsearch.index.fielddata.HistogramValue; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.analytics.aggregations.support.HistogramValuesSource; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +abstract class AbstractHistoBackedHDRPercentilesAggregator extends NumericMetricsAggregator.MultiValue { + + private static int indexOfKey(double[] keys, double key) { + return ArrayUtils.binarySearch(keys, key, 0.001); + } + + protected final double[] keys; + protected final ValuesSource valuesSource; + protected final DocValueFormat format; + protected ObjectArray states; + protected final int numberOfSignificantValueDigits; + protected final boolean keyed; + + AbstractHistoBackedHDRPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] keys, int numberOfSignificantValueDigits, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) throws IOException { + super(name, context, parent, pipelineAggregators, metaData); + this.valuesSource = valuesSource; + this.keyed = keyed; + this.format = formatter; + this.states = context.bigArrays().newObjectArray(1); + this.keys = keys; + this.numberOfSignificantValueDigits = numberOfSignificantValueDigits; + } + + @Override + public ScoreMode scoreMode() { + return valuesSource != null && valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, + final LeafBucketCollector sub) throws IOException { + if (valuesSource == null) { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + final BigArrays bigArrays = context.bigArrays(); + final HistogramValues values = ((HistogramValuesSource.Histogram)valuesSource).getHistogramValues(ctx); + + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + DoubleHistogram state = getExistingOrNewHistogram(bigArrays, bucket); + if (values.advanceExact(doc)) { + final HistogramValue sketch = values.histogram(); + while (sketch.next()) { + state.recordValueWithCount(sketch.value(), sketch.count()); + } + } + } + }; + } + + private DoubleHistogram getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) { + states = bigArrays.grow(states, bucket + 1); + DoubleHistogram state = states.get(bucket); + if (state == null) { + state = new DoubleHistogram(numberOfSignificantValueDigits); + /* Set the histogram to autosize so it can resize itself as + the data range increases. Resize operations should be + rare as the histogram buckets are exponential (on the top + level). In the future we could expose the range as an + option on the request so the histogram can be fixed at + initialisation and doesn't need resizing. + */ + state.setAutoResize(true); + states.set(bucket, state); + } + return state; + } + + @Override + public boolean hasMetric(String name) { + return indexOfKey(keys, Double.parseDouble(name)) >= 0; + } + + protected DoubleHistogram getState(long bucketOrd) { + if (bucketOrd >= states.size()) { + return null; + } + final DoubleHistogram state = states.get(bucketOrd); + return state; + } + + @Override + protected void doClose() { + Releasables.close(states); + } + +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedTDigestPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedTDigestPercentilesAggregator.java new file mode 100644 index 0000000000000..f4ae77c95b536 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedTDigestPercentilesAggregator.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ScoreMode; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.util.ArrayUtils; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ObjectArray; +import org.elasticsearch.index.fielddata.HistogramValue; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.analytics.aggregations.support.HistogramValuesSource; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +abstract class AbstractHistoBackedTDigestPercentilesAggregator extends NumericMetricsAggregator.MultiValue { + + private static int indexOfKey(double[] keys, double key) { + return ArrayUtils.binarySearch(keys, key, 0.001); + } + + protected final double[] keys; + protected final ValuesSource valuesSource; + protected final DocValueFormat formatter; + protected ObjectArray states; + protected final double compression; + protected final boolean keyed; + + AbstractHistoBackedTDigestPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] keys, double compression, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) throws IOException { + super(name, context, parent, pipelineAggregators, metaData); + this.valuesSource = valuesSource; + this.keyed = keyed; + this.formatter = formatter; + this.states = context.bigArrays().newObjectArray(1); + this.keys = keys; + this.compression = compression; + } + + @Override + public ScoreMode scoreMode() { + return valuesSource != null && valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, + final LeafBucketCollector sub) throws IOException { + if (valuesSource == null) { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + final BigArrays bigArrays = context.bigArrays(); + final HistogramValues values = ((HistogramValuesSource.Histogram)valuesSource).getHistogramValues(ctx); + + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + TDigestState state = getExistingOrNewHistogram(bigArrays, bucket); + if (values.advanceExact(doc)) { + final HistogramValue sketch = values.histogram(); + while(sketch.next()) { + state.add(sketch.value(), sketch.count()); + } + } + } + }; + } + + private TDigestState getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) { + states = bigArrays.grow(states, bucket + 1); + TDigestState state = states.get(bucket); + if (state == null) { + state = new TDigestState(compression); + states.set(bucket, state); + } + return state; + } + + @Override + public boolean hasMetric(String name) { + return indexOfKey(keys, Double.parseDouble(name)) >= 0; + } + + protected TDigestState getState(long bucketOrd) { + if (bucketOrd >= states.size()) { + return null; + } + final TDigestState state = states.get(bucketOrd); + return state; + } + + @Override + protected void doClose() { + Releasables.close(states); + } + +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsPercentilesAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsPercentilesAggregatorFactory.java new file mode 100644 index 0000000000000..2a50482d5f59f --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsPercentilesAggregatorFactory.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesAggregatorSupplier; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; +import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; + +public class AnalyticsPercentilesAggregatorFactory { + public static void registerPercentilesAggregator(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(PercentilesAggregationBuilder.NAME, + AnalyticsValuesSourceType.HISTOGRAM, + (PercentilesAggregatorSupplier) (name, valuesSource, context, parent, percents, percentilesConfig, keyed, + formatter, pipelineAggregators, metaData) -> { + + if (percentilesConfig.getMethod().equals(PercentilesMethod.TDIGEST)) { + double compression = ((PercentilesConfig.TDigest)percentilesConfig).getCompression(); + return new HistoBackedTDigestPercentilesAggregator(name, valuesSource, context, parent, + percents, compression, keyed, formatter, pipelineAggregators, metaData); + + } else if (percentilesConfig.getMethod().equals(PercentilesMethod.HDR)) { + int numSigFig = ((PercentilesConfig.Hdr)percentilesConfig).getNumberOfSignificantValueDigits(); + return new HistoBackedHDRPercentilesAggregator(name, valuesSource, context, parent, + percents, numSigFig, keyed, formatter, pipelineAggregators, metaData); + } + + throw new IllegalArgumentException("Percentiles algorithm: [" + percentilesConfig.getMethod().toString() + "] " + + "is not compatible with Histogram field"); + }); + } + + public static void registerPercentileRanksAggregator(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(PercentileRanksAggregationBuilder.NAME, + AnalyticsValuesSourceType.HISTOGRAM, + (PercentilesAggregatorSupplier) (name, valuesSource, context, parent, percents, percentilesConfig, keyed, + formatter, pipelineAggregators, metaData) -> { + + if (percentilesConfig.getMethod().equals(PercentilesMethod.TDIGEST)) { + double compression = ((PercentilesConfig.TDigest)percentilesConfig).getCompression(); + return new HistoBackedTDigestPercentileRanksAggregator(name, valuesSource, context, parent, + percents, compression, keyed, formatter, pipelineAggregators, metaData); + + } else if (percentilesConfig.getMethod().equals(PercentilesMethod.HDR)) { + int numSigFig = ((PercentilesConfig.Hdr)percentilesConfig).getNumberOfSignificantValueDigits(); + return new HistoBackedHDRPercentileRanksAggregator(name, valuesSource, context, parent, + percents, numSigFig, keyed, formatter, pipelineAggregators, metaData); + } + + throw new IllegalArgumentException("Percentiles algorithm: [" + percentilesConfig.getMethod().toString() + "] " + + "is not compatible with Histogram field"); + }); + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java new file mode 100644 index 0000000000000..dc9bca57ec3cd --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.HdrHistogram.DoubleHistogram; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentileRanks; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +class HistoBackedHDRPercentileRanksAggregator extends AbstractHistoBackedHDRPercentilesAggregator { + + HistoBackedHDRPercentileRanksAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] percents, int numberOfSignificantValueDigits, boolean keyed, DocValueFormat format, + List pipelineAggregators, Map metaData) throws IOException { + super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, format, pipelineAggregators, + metaData); + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + DoubleHistogram state = getState(owningBucketOrdinal); + if (state == null) { + return buildEmptyAggregation(); + } else { + return new InternalHDRPercentileRanks(name, keys, state, keyed, format, pipelineAggregators(), metaData()); + } + } + + @Override + public InternalAggregation buildEmptyAggregation() { + DoubleHistogram state; + state = new DoubleHistogram(numberOfSignificantValueDigits); + state.setAutoResize(true); + return new InternalHDRPercentileRanks(name, keys, state, + keyed, format, pipelineAggregators(), metaData()); + } + + @Override + public double metric(String name, long bucketOrd) { + DoubleHistogram state = getState(bucketOrd); + if (state == null) { + return Double.NaN; + } else { + return InternalHDRPercentileRanks.percentileRank(state, Double.valueOf(name)); + } + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java new file mode 100644 index 0000000000000..283f07fcc71f4 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.HdrHistogram.DoubleHistogram; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentiles; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class HistoBackedHDRPercentilesAggregator extends AbstractHistoBackedHDRPercentilesAggregator { + + HistoBackedHDRPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, double[] percents, + int numberOfSignificantValueDigits, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) throws IOException { + super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, formatter, + pipelineAggregators, metaData); + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + DoubleHistogram state = getState(owningBucketOrdinal); + if (state == null) { + return buildEmptyAggregation(); + } else { + return new InternalHDRPercentiles(name, keys, state, keyed, format, pipelineAggregators(), metaData()); + } + } + + @Override + public double metric(String name, long bucketOrd) { + DoubleHistogram state = getState(bucketOrd); + if (state == null) { + return Double.NaN; + } else { + return state.getValueAtPercentile(Double.parseDouble(name)); + } + } + + @Override + public InternalAggregation buildEmptyAggregation() { + DoubleHistogram state; + state = new DoubleHistogram(numberOfSignificantValueDigits); + state.setAutoResize(true); + return new InternalHDRPercentiles(name, keys, state, + keyed, + format, pipelineAggregators(), metaData()); + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java new file mode 100644 index 0000000000000..7bd1f9d35e995 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +class HistoBackedTDigestPercentileRanksAggregator extends AbstractHistoBackedTDigestPercentilesAggregator { + + HistoBackedTDigestPercentileRanksAggregator(String name, + ValuesSource valuesSource, + SearchContext context, + Aggregator parent, + double[] percents, + double compression, + boolean keyed, + DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException { + super(name, valuesSource, context, parent, percents, compression, keyed, formatter, pipelineAggregators, metaData); + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + TDigestState state = getState(owningBucketOrdinal); + if (state == null) { + return buildEmptyAggregation(); + } else { + return new InternalTDigestPercentileRanks(name, keys, state, keyed, formatter, pipelineAggregators(), metaData()); + } + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalTDigestPercentileRanks(name, keys, new TDigestState(compression), keyed, + formatter, pipelineAggregators(), metaData()); + } + + @Override + public double metric(String name, long bucketOrd) { + TDigestState state = getState(bucketOrd); + if (state == null) { + return Double.NaN; + } else { + return InternalTDigestPercentileRanks.percentileRank(state, Double.valueOf(name)); + } + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java new file mode 100644 index 0000000000000..6b7f33178ebf1 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +class HistoBackedTDigestPercentilesAggregator extends AbstractHistoBackedTDigestPercentilesAggregator { + + HistoBackedTDigestPercentilesAggregator(String name, + ValuesSource valuesSource, + SearchContext context, + Aggregator parent, + double[] percents, + double compression, + boolean keyed, + DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException { + super(name, valuesSource, context, parent, percents, compression, keyed, formatter, pipelineAggregators, metaData); + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + TDigestState state = getState(owningBucketOrdinal); + if (state == null) { + return buildEmptyAggregation(); + } else { + return new InternalTDigestPercentiles(name, keys, state, keyed, formatter, pipelineAggregators(), metaData()); + } + } + + @Override + public double metric(String name, long bucketOrd) { + TDigestState state = getState(bucketOrd); + if (state == null) { + return Double.NaN; + } else { + return state.quantile(Double.parseDouble(name) / 100); + } + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalTDigestPercentiles(name, keys, new TDigestState(compression), keyed, + formatter, pipelineAggregators(), metaData()); + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java new file mode 100644 index 0000000000000..8ff07fda257b5 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.analytics.aggregations.support; + +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexHistogramFieldData; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; +import org.elasticsearch.search.aggregations.support.FieldContext; +import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + +import java.util.Locale; +import java.util.function.LongSupplier; + +public enum AnalyticsValuesSourceType implements ValuesSourceType { + HISTOGRAM() { + @Override + public ValuesSource getEmpty() { + // TODO: Is this the correct exception type here? + throw new IllegalArgumentException("Can't deal with unmapped HistogramValuesSource type " + this.value()); + } + + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + throw new AggregationExecutionException("value source of type [" + this.value() + "] is not supported by scripts"); + } + + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + final IndexFieldData indexFieldData = fieldContext.indexFieldData(); + + if (!(indexFieldData instanceof IndexHistogramFieldData)) { + throw new IllegalArgumentException("Expected histogram type on field [" + fieldContext.field() + + "], but got [" + fieldContext.fieldType().typeName() + "]"); + } + return new HistogramValuesSource.Histogram.Fielddata((IndexHistogramFieldData) indexFieldData); + } + + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + throw new IllegalArgumentException("Can't apply missing values on a " + valuesSource.getClass()); + } + }; + + + public static ValuesSourceType fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public String value() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java new file mode 100644 index 0000000000000..54a9582b4f34b --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.support; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.DocValueBits; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.index.fielddata.IndexHistogramFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; + +import java.io.IOException; + +public class HistogramValuesSource { + public abstract static class Histogram extends org.elasticsearch.search.aggregations.support.ValuesSource { + + public abstract HistogramValues getHistogramValues(LeafReaderContext context) throws IOException; + + public static class Fielddata extends Histogram { + + protected final IndexHistogramFieldData indexFieldData; + + public Fielddata(IndexHistogramFieldData indexFieldData) { + this.indexFieldData = indexFieldData; + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) { + return indexFieldData.load(context).getBytesValues(); + } + + @Override + public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { + HistogramValues values = getHistogramValues(context); + return new DocValueBits() { + @Override + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); + } + }; + } + + public HistogramValues getHistogramValues(LeafReaderContext context) throws IOException { + return indexFieldData.load(context).getHistogramValues(); + } + } + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java index f10a0ba09235b..677b6d0cf4223 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java @@ -16,11 +16,11 @@ import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; @@ -35,14 +35,14 @@ public class BoxplotAggregationBuilder extends ValuesSourceAggregationBuilder.Le public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, BoxplotAggregationBuilder::new); static { - ValuesSourceParserHelper.declareAnyFields(PARSER, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); PARSER.declareDouble(BoxplotAggregationBuilder::compression, COMPRESSION_FIELD); } private double compression = 100.0; public BoxplotAggregationBuilder(String name) { - super(name, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(name); } protected BoxplotAggregationBuilder(BoxplotAggregationBuilder clone, @@ -51,6 +51,10 @@ protected BoxplotAggregationBuilder(BoxplotAggregationBuilder clone, this.compression = clone.compression; } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + BoxplotAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + @Override protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metaData) { return new BoxplotAggregationBuilder(this, factoriesBuilder, metaData); @@ -60,7 +64,7 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu * Read from a stream. */ public BoxplotAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(in); compression = in.readDouble(); } @@ -69,6 +73,11 @@ protected void innerWriteTo(StreamOutput out) throws IOException { out.writeDouble(compression); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.NUMERIC; + } + /** * Expert: set the compression. Higher values improve accuracy but also * memory usage. Only relevant when using {@link PercentilesMethod#TDIGEST}. @@ -92,7 +101,7 @@ public double compression() { @Override protected BoxplotAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, + ValuesSourceConfig config, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException { return new BoxplotAggregatorFactory(name, config, compression, queryShardContext, parent, subFactoriesBuilder, metaData); diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregator.java index dec42ddbd6159..ec07e1642830e 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregator.java @@ -24,6 +24,7 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.analytics.aggregations.support.HistogramValuesSource; import java.io.IOException; import java.util.List; @@ -60,8 +61,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - if (valuesSource instanceof ValuesSource.Histogram) { - final HistogramValues values = ((ValuesSource.Histogram)valuesSource).getHistogramValues(ctx); + if (valuesSource instanceof HistogramValuesSource.Histogram) { + final HistogramValues values = ((HistogramValuesSource.Histogram)valuesSource).getHistogramValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java index ac34667a9e173..975cb34465be1 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java @@ -7,25 +7,36 @@ package org.elasticsearch.xpack.analytics.boxplot; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import java.io.IOException; import java.util.List; import java.util.Map; -public class BoxplotAggregatorFactory extends ValuesSourceAggregatorFactory { +public class BoxplotAggregatorFactory extends ValuesSourceAggregatorFactory { private final double compression; + static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(BoxplotAggregationBuilder.NAME, + List.of(CoreValuesSourceType.NUMERIC, AnalyticsValuesSourceType.HISTOGRAM), + (BoxplotAggregatorSupplier) BoxplotAggregator::new); + } + BoxplotAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, double compression, QueryShardContext queryShardContext, AggregatorFactory parent, @@ -52,8 +63,14 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new BoxplotAggregator(name, valuesSource, config.format(), compression, searchContext, parent, - pipelineAggregators, metaData); - } + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + BoxplotAggregationBuilder.NAME); + if (aggregatorSupplier instanceof BoxplotAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected BoxplotAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((BoxplotAggregatorSupplier) aggregatorSupplier).build(name, valuesSource, config.format(), compression, + searchContext, parent, pipelineAggregators, metaData); + } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorSupplier.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorSupplier.java new file mode 100644 index 0000000000000..3555014b8e9ca --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorSupplier.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.boxplot; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@FunctionalInterface +public interface BoxplotAggregatorSupplier extends AggregatorSupplier { + Aggregator build(String name, + ValuesSource valuesSource, + DocValueFormat formatter, + double compression, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException; + +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 6b77018c11638..ceab5dc8dfb1e 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -34,11 +34,11 @@ import org.elasticsearch.index.fielddata.HistogramValue; import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -51,10 +51,10 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import java.io.IOException; import java.util.Iterator; @@ -280,8 +280,7 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu @Override public ValuesSourceType getValuesSourceType() { - // TODO: Histogram ValuesSourceType should move into this plugin. - return CoreValuesSourceType.HISTOGRAM; + return AnalyticsValuesSourceType.HISTOGRAM; } @Override diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java index 8602bfb6eec9d..e12cd047234eb 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java @@ -15,31 +15,31 @@ import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; import java.util.Objects; -public class StringStatsAggregationBuilder extends ValuesSourceAggregationBuilder { +public class StringStatsAggregationBuilder extends ValuesSourceAggregationBuilder { + public static final String NAME = "string_stats"; private static final ParseField SHOW_DISTRIBUTION_FIELD = new ParseField("show_distribution"); public static final ObjectParser PARSER = ObjectParser.fromBuilder(NAME, StringStatsAggregationBuilder::new); static { - ValuesSourceParserHelper.declareBytesFields(PARSER, true, true); + ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); PARSER.declareBoolean(StringStatsAggregationBuilder::showDistribution, StringStatsAggregationBuilder.SHOW_DISTRIBUTION_FIELD); } private boolean showDistribution = false; public StringStatsAggregationBuilder(String name) { - super(name, CoreValuesSourceType.BYTES, ValueType.STRING); + super(name); } public StringStatsAggregationBuilder(StringStatsAggregationBuilder clone, @@ -51,10 +51,15 @@ public StringStatsAggregationBuilder(StringStatsAggregationBuilder clone, /** Read from a stream. */ public StringStatsAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.BYTES, ValueType.STRING); + super(in); this.showDistribution = in.readBoolean(); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metaData) { return new StringStatsAggregationBuilder(this, factoriesBuilder, metaData); @@ -67,7 +72,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException { @Override protected StringStatsAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, + ValuesSourceConfig config, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException { return new StringStatsAggregatorFactory(name, config, showDistribution, queryShardContext, parent, subFactoriesBuilder, metaData); @@ -103,6 +108,10 @@ public StringStatsAggregationBuilder showDistribution(boolean showDistribution) return this; } + public static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { + StringStatsAggregatorFactory.registerAggregators(valuesSourceRegistry); + } + @Override public int hashCode() { return Objects.hash(super.hashCode(), showDistribution); diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorFactory.java index 5a80bc0f58969..7ab050a32c6de 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorFactory.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorFactory.java @@ -7,24 +7,29 @@ package org.elasticsearch.xpack.analytics.stringstats; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.List; import java.util.Map; -class StringStatsAggregatorFactory extends ValuesSourceAggregatorFactory { +class StringStatsAggregatorFactory extends ValuesSourceAggregatorFactory { private final boolean showDistribution; - StringStatsAggregatorFactory(String name, ValuesSourceConfig config, + StringStatsAggregatorFactory(String name, ValuesSourceConfig config, Boolean showDistribution, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) @@ -33,6 +38,24 @@ class StringStatsAggregatorFactory extends ValuesSourceAggregatorFactory pipelineAggregators, + Map metaData) throws IOException { + return new StringStatsAggregator(name, showDistribution, (ValuesSource.Bytes) valuesSource, + format, context, parent, pipelineAggregators, metaData); + } + }); + } + @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, @@ -43,14 +66,21 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(ValuesSource.Bytes valuesSource, + protected Aggregator doCreateInternal(ValuesSource valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - return new StringStatsAggregator(name, showDistribution, valuesSource, config.format(), searchContext, parent, - pipelineAggregators, metaData); + AggregatorSupplier aggregatorSupplier = queryShardContext.getValuesSourceRegistry().getAggregator(config.valueSourceType(), + StringStatsAggregationBuilder.NAME); + + if (aggregatorSupplier instanceof StringStatsAggregatorSupplier == false) { + throw new AggregationExecutionException("Registry miss-match - expected StringStatsAggregatorSupplier, found [" + + aggregatorSupplier.getClass().toString() + "]"); + } + return ((StringStatsAggregatorSupplier) aggregatorSupplier).build(name, valuesSource, showDistribution, config.format(), + searchContext, parent, pipelineAggregators, metaData); } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorSupplier.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorSupplier.java new file mode 100644 index 0000000000000..d996ec0b309a1 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorSupplier.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.analytics.stringstats; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.AggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface StringStatsAggregatorSupplier extends AggregatorSupplier { + + Aggregator build(String name, + ValuesSource valuesSource, + boolean showDistribution, + DocValueFormat format, + SearchContext context, + Aggregator parent, + List pipelineAggregators, + Map metaData) throws IOException; +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java index 2c2599e450a97..75556033f26ae 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorFactory.java @@ -13,6 +13,7 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; @@ -58,11 +59,12 @@ protected TopMetricsAggregator createInternal(SearchContext searchContext, Aggre + "] index level setting."); } List metricSources = metricFields.stream().map(config -> { - ValuesSourceConfig resolved = ValuesSourceConfig.resolve( + ValuesSourceConfig resolved = ValuesSourceConfig.resolve( searchContext.getQueryShardContext(), ValueType.NUMERIC, - config.getFieldName(), config.getScript(), config.getMissing(), config.getTimeZone(), null); + config.getFieldName(), config.getScript(), config.getMissing(), config.getTimeZone(), null, + CoreValuesSourceType.NUMERIC, TopMetricsAggregationBuilder.NAME); return new TopMetricsAggregator.MetricSource(config.getFieldName(), resolved.format(), - resolved.toValuesSource(searchContext.getQueryShardContext())); + (ValuesSource.Numeric) resolved.toValuesSource()); }).collect(toList()); return new TopMetricsAggregator(name, searchContext, parent, pipelineAggregators, metaData, size, sortBuilders.get(0), metricSources); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java index d8af20e311628..8245f6c529221 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java @@ -39,6 +39,8 @@ import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Collections; @@ -56,6 +58,12 @@ public class BoxplotAggregatorTests extends AggregatorTestCase { /** Script to return the {@code _value} provided by aggs framework. */ public static final String VALUE_SCRIPT = "_value"; + + @BeforeClass() + public static void registerBuilder() { + BoxplotAggregationBuilder.registerAggregators(valuesSourceRegistry); + } + @Override protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { return new BoxplotAggregationBuilder("foo").field(fieldName); @@ -63,8 +71,7 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.NUMERIC, - CoreValuesSourceType.HISTOGRAM); + return List.of(CoreValuesSourceType.NUMERIC, AnalyticsValuesSourceType.HISTOGRAM); } @Override @@ -81,7 +88,6 @@ protected ScriptService getMockScriptService() { return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); } - public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7))); @@ -219,7 +225,8 @@ public void testUnsupportedType() { }, (Consumer) boxplot -> { fail("Should have thrown exception"); }, fieldType)); - assertEquals(e.getMessage(), "Expected numeric type on field [not_a_number], but got [keyword]"); + assertEquals(e.getMessage(), "Field [not_a_number] of type [keyword(indexed,tokenized)] " + + "is not supported for aggregation [boxplot]"); } public void testBadMissingField() { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java index fd7adddd4234c..a8894e3c13a92 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; -import org.elasticsearch.search.aggregations.support.ValueType; import java.io.IOException; import java.util.Arrays; @@ -64,7 +63,7 @@ public void testSimple() throws IOException { DateHistogramAggregationBuilder aggBuilder = new DateHistogramAggregationBuilder("histo"); aggBuilder.calendarInterval(DateHistogramInterval.DAY).field(HISTO_FIELD); - aggBuilder.subAggregation(new CardinalityAggregationBuilder("the_cardinality", ValueType.NUMERIC).field(VALUE_FIELD)); + aggBuilder.subAggregation(new CardinalityAggregationBuilder("the_cardinality").field(VALUE_FIELD)); aggBuilder.subAggregation(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "the_cardinality")); executeTestCase(query, aggBuilder, histogram -> { @@ -84,7 +83,7 @@ public void testAllNull() throws IOException { DateHistogramAggregationBuilder aggBuilder = new DateHistogramAggregationBuilder("histo"); aggBuilder.calendarInterval(DateHistogramInterval.DAY).field(HISTO_FIELD); - aggBuilder.subAggregation(new CardinalityAggregationBuilder("the_cardinality", ValueType.NUMERIC).field("foo")); + aggBuilder.subAggregation(new CardinalityAggregationBuilder("the_cardinality").field("foo")); aggBuilder.subAggregation(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "the_cardinality")); executeTestCase(query, aggBuilder, histogram -> { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentileRanksAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentileRanksAggregatorTests.java index 843ff9447fde5..fab7bd91e0ea3 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentileRanksAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentileRanksAggregatorTests.java @@ -16,20 +16,49 @@ import org.apache.lucene.store.Directory; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.elasticsearch.search.aggregations.metrics.Percentile; import org.elasticsearch.search.aggregations.metrics.PercentileRanks; import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.hamcrest.Matchers; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Iterator; +import java.util.List; public class HDRPreAggregatedPercentileRanksAggregatorTests extends AggregatorTestCase { + @BeforeClass + public static void registerBuilder() { + AnalyticsPercentilesAggregatorFactory.registerPercentileRanksAggregator(valuesSourceRegistry); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentileRanksAggregationBuilder("hdr_percentiles", new double[]{1.0}) + .field(fieldName) + .percentilesConfig(new PercentilesConfig.Hdr()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + // Note: this is the same list as Core, plus Analytics + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + AnalyticsValuesSourceType.HISTOGRAM); + } + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { DoubleHistogram histogram = new DoubleHistogram(3);//default for (double value : values) { @@ -62,7 +91,7 @@ public void testSimple() throws IOException { fieldType.setName("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentilesAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentilesAggregatorTests.java index 5d4e5c05b2e9d..d71797039d78d 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentilesAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentilesAggregatorTests.java @@ -19,22 +19,51 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentiles; import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; public class HDRPreAggregatedPercentilesAggregatorTests extends AggregatorTestCase { - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { + @BeforeClass + public static void registerBuilder() { + AnalyticsPercentilesAggregatorFactory.registerPercentilesAggregator(valuesSourceRegistry); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentilesAggregationBuilder("hdr_percentiles") + .field(fieldName) + .percentilesConfig(new PercentilesConfig.Hdr()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + // Note: this is the same list as Core, plus Analytics + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + AnalyticsValuesSourceType.HISTOGRAM); + } + + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { DoubleHistogram histogram = new DoubleHistogram(3);//default for (double value : values) { histogram.recordValue(value); @@ -54,7 +83,7 @@ private BinaryDocValuesField getDocValue(String fieldName, double[] values) thro } return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } + } public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentileRanksAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentileRanksAggregatorTests.java index 879173a5bc26e..ad80192a3c67b 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentileRanksAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentileRanksAggregatorTests.java @@ -16,23 +16,52 @@ import org.apache.lucene.store.Directory; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.Percentile; import org.elasticsearch.search.aggregations.metrics.PercentileRanks; import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.hamcrest.Matchers; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Collection; import java.util.Iterator; +import java.util.List; public class TDigestPreAggregatedPercentileRanksAggregatorTests extends AggregatorTestCase { + @BeforeClass + public static void registerBuilder() { + AnalyticsPercentilesAggregatorFactory.registerPercentileRanksAggregator(valuesSourceRegistry); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentileRanksAggregationBuilder("tdigest_percentiles", new double[]{1.0}) + .field(fieldName) + .percentilesConfig(new PercentilesConfig.TDigest()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + // Note: this is the same list as Core, plus Analytics + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + AnalyticsValuesSourceType.HISTOGRAM); + } + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { TDigest histogram = new TDigestState(100.0); //default for (double value : values) { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentilesAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentilesAggregatorTests.java index e1340619256cf..6b084535addb8 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentilesAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentilesAggregatorTests.java @@ -19,24 +19,53 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles; import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTestCase { - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { + @BeforeClass + public static void registerBuilder() { + AnalyticsPercentilesAggregatorFactory.registerPercentilesAggregator(valuesSourceRegistry); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentilesAggregationBuilder("tdigest_percentiles") + .field(fieldName) + .percentilesConfig(new PercentilesConfig.TDigest()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + // Note: this is the same list as Core, plus Analytics + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + AnalyticsValuesSourceType.HISTOGRAM); + } + + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { TDigest histogram = new TDigestState(100.0); //default for (double value : values) { histogram.add(value); @@ -51,7 +80,7 @@ private BinaryDocValuesField getDocValue(String fieldName, double[] values) thro streamOutput.writeDouble(centroid.mean()); } return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } + } public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java index c9790638da078..d975f56bd919b 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValueType; +import org.junit.BeforeClass; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.lookup.LeafDocLookup; @@ -55,6 +56,10 @@ public class StringStatsAggregatorTests extends AggregatorTestCase { + @BeforeClass() + public static void registerBuilder() { + StringStatsAggregationBuilder.registerAggregators(valuesSourceRegistry); + } private static final String VALUE_SCRIPT_NAME = "value_script"; private static final String FIELD_SCRIPT_NAME = "field_script"; @@ -254,9 +259,10 @@ public void testNestedAggregation() throws IOException { textFieldType.setName("text"); textFieldType.setFielddata(true); - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("terms", ValueType.NUMERIC) + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("terms") + .userValueTypeHint(ValueType.NUMERIC) .field("value") - .subAggregation(new StringStatsAggregationBuilder("text_stats").field("text").valueType(ValueType.STRING)); + .subAggregation(new StringStatsAggregationBuilder("text_stats").field("text").userValueTypeHint(ValueType.STRING)); Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index 39dc9bf5e9fda..c13b4a94ade54 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -55,7 +55,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.GeoDistanceSortBuilder; @@ -283,14 +282,14 @@ private InternalTopMetrics collectFromNewYorkAndLA(TopMetricsAggregationBuilder public void testSortByGeoDistancDescending() throws IOException { TopMetricsAggregationBuilder builder = simpleBuilder(new GeoDistanceSortBuilder("s", 35.7796, 78.6382).order(SortOrder.DESC)); - InternalTopMetrics result = collectFromNewYorkAndLA(builder); + InternalTopMetrics result = collectFromNewYorkAndLA(builder); assertThat(result.getSortOrder(), equalTo(SortOrder.DESC)); assertThat(result.getTopMetrics(), equalTo(singletonList(top(1.2054632268631617E7, 3.0)))); } public void testSortByGeoDistanceAscending() throws IOException { TopMetricsAggregationBuilder builder = simpleBuilder(new GeoDistanceSortBuilder("s", 35.7796, 78.6382).order(SortOrder.ASC)); - InternalTopMetrics result = collectFromNewYorkAndLA(builder); + InternalTopMetrics result = collectFromNewYorkAndLA(builder); assertThat(result.getSortOrder(), equalTo(SortOrder.ASC)); assertThat(result.getTopMetrics(), equalTo(singletonList(top(1.1062351376961706E7, 2.0)))); } @@ -305,7 +304,7 @@ public void testSortByGeoDistanceTwoHits() throws IOException { public void testInsideTerms() throws IOException { TopMetricsAggregationBuilder builder = simpleBuilder(new FieldSortBuilder("s").order(SortOrder.ASC)); - TermsAggregationBuilder terms = new TermsAggregationBuilder("terms", ValueType.DOUBLE).field("c").subAggregation(builder); + TermsAggregationBuilder terms = new TermsAggregationBuilder("terms").field("c").subAggregation(builder); Terms result = (Terms) collect(terms, new MatchAllDocsQuery(), writer -> { writer.addDocument(Arrays.asList(doubleField("c", 1.0), doubleField("s", 1.0), doubleField("m", 2.0))); writer.addDocument(Arrays.asList(doubleField("c", 1.0), doubleField("s", 2.0), doubleField("m", 3.0))); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java index a6a1feaf133f5..f22349008e0f2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java @@ -70,7 +70,7 @@ public class RollupField { * @param extra The type of value this field is (VALUE, INTERVAL, etc) * @return formatted field name */ - public static String formatFieldName(ValuesSourceAggregationBuilder source, String extra) { + public static String formatFieldName(ValuesSourceAggregationBuilder source, String extra) { return source.field() + "." + source.getType() + "." + extra; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index d6097f06d5fa7..a51af35e8f618 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -523,7 +523,7 @@ private TestIndexContext testIndex(MapperService mapperService, Client client) t final QueryShardContext shardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), - client, new IndexSearcher(directoryReader), () -> nowInMillis, null, null, () -> true); + client, new IndexSearcher(directoryReader), () -> nowInMillis, null, null, () -> true, null); context = new TestIndexContext(directory, iw, directoryReader, shardContext, leaf); return context; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index 422e5dd0128ba..e7a29c7083824 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -93,7 +93,7 @@ public void testDLS() throws Exception { final long nowInMillis = randomNonNegativeLong(); QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), - client, null, () -> nowInMillis, null, null, () -> true); + client, null, () -> nowInMillis, null, null, () -> true, null); QueryShardContext queryShardContext = spy(realQueryShardContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY, Executors.newSingleThreadExecutor()); XPackLicenseState licenseState = mock(XPackLicenseState.class); @@ -227,7 +227,7 @@ public void testDLSWithLimitedPermissions() throws Exception { final long nowInMillis = randomNonNegativeLong(); QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), - client, null, () -> nowInMillis, null, null, () -> true); + client, null, () -> nowInMillis, null, null, () -> true, null); QueryShardContext queryShardContext = spy(realQueryShardContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY, Executors.newSingleThreadExecutor()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java index 8f04e8ff110e4..ed5a49efa487e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java @@ -17,16 +17,15 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Map; -public class MockDeprecatedAggregationBuilder extends ValuesSourceAggregationBuilder { +public class MockDeprecatedAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "deprecated_agg"; public static final String DEPRECATION_MESSAGE = "expected deprecation message from MockDeprecatedAggregationBuilder"; @@ -39,20 +38,25 @@ protected MockDeprecatedAggregationBuilder(MockDeprecatedAggregationBuilder clon super(clone, factoriesBuilder, metaData); } + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new MockDeprecatedAggregationBuilder(this, factoriesBuilder, metaData); } public MockDeprecatedAggregationBuilder() { - super(NAME, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC); + super(NAME); } /** * Read from a stream. */ protected MockDeprecatedAggregationBuilder(StreamInput in) throws IOException { - super(in, null, null); + super(in); } @Override @@ -65,10 +69,10 @@ protected void innerWriteTo(StreamOutput out) throws IOException { } @Override - protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { return null; } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 115968b69ee65..9ffcdbcfdc21f 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -43,6 +43,8 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; /** * A {@link FieldMapper} that assigns every document the same value. @@ -248,6 +250,10 @@ public Query regexpQuery(String value, int flags, int maxDeterminizedStates, } } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } } ConstantKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 9bbb2a5d6d6c1..5d5ed7bd48e8e 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -53,10 +53,10 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.sort.BucketedSort; -import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; import java.io.IOException; import java.util.Iterator; @@ -541,6 +541,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); return new DocValuesIndexFieldData.Builder(); } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } } private FlatObjectFieldParser fieldParser; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java index d66936957e071..a8e2b70863ebe 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java @@ -118,7 +118,7 @@ public static void create(Client client, ); return; } - final List> flattenedAggs = new ArrayList<>(); + final List> flattenedAggs = new ArrayList<>(); flattenAggregations(datafeed.getParsedAggregations(xContentRegistry) .getAggregatorFactories(), datafeedHistogramAggregation, flattenedAggs); @@ -149,7 +149,7 @@ private static boolean validInterval(long datafeedInterval, ParsedRollupCaps rol private static void flattenAggregations(final Collection datafeedAggregations, final AggregationBuilder datafeedHistogramAggregation, - final List> flattenedAggregations) { + final List> flattenedAggregations) { for (AggregationBuilder aggregationBuilder : datafeedAggregations) { if (aggregationBuilder.equals(datafeedHistogramAggregation) == false) { flattenedAggregations.add((ValuesSourceAggregationBuilder)aggregationBuilder); @@ -158,8 +158,8 @@ private static void flattenAggregations(final Collection dat } } - private static boolean hasAggregations(ParsedRollupCaps rollupCaps, List> datafeedAggregations) { - for (ValuesSourceAggregationBuilder aggregationBuilder : datafeedAggregations) { + private static boolean hasAggregations(ParsedRollupCaps rollupCaps, List> datafeedAggregations) { + for (ValuesSourceAggregationBuilder aggregationBuilder : datafeedAggregations) { String type = aggregationBuilder.getType(); String field = aggregationBuilder.field(); if (aggregationBuilder instanceof TermsAggregationBuilder) { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java index b610dca45086e..a5795b90f9b01 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java @@ -319,7 +319,10 @@ private static List translateTerms(TermsAggregationBuilder s return translateVSAggBuilder(source, registry, () -> { TermsAggregationBuilder rolledTerms - = new TermsAggregationBuilder(source.getName(), source.valueType()); + = new TermsAggregationBuilder(source.getName()); + if (source.userValueTypeHint() != null) { + rolledTerms.userValueTypeHint(source.userValueTypeHint()); + } rolledTerms.field(RollupField.formatFieldName(source, RollupField.VALUE)); rolledTerms.includeExclude(source.includeExclude()); if (source.collectMode() != null) { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java index 9bd5e5b6ff9b9..b70b355665e23 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java @@ -26,7 +26,6 @@ import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; -import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xpack.core.indexing.AsyncTwoPhaseIndexer; @@ -282,16 +281,17 @@ static List createAggregationBuilders(final List caps = new HashSet<>(2); caps.add(cap); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java index cd169c1ae7ce1..0ca0cbb16ddbc 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java @@ -62,7 +62,7 @@ public void testBasicDateHisto() { .extendedBounds(new ExtendedBounds(0L, 1000L)) .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")) .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field")); - + List translated = translateAggregation(histo, namedWriteableRegistry); assertThat(translated.size(), equalTo(1)); assertThat(translated.get(0), Matchers.instanceOf(DateHistogramAggregationBuilder.class)); @@ -302,7 +302,7 @@ public void testAvgMetric() { public void testStringTerms() throws IOException { - TermsAggregationBuilder terms = new TermsAggregationBuilder("test_string_terms", ValueType.STRING); + TermsAggregationBuilder terms = new TermsAggregationBuilder("test_string_terms").userValueTypeHint(ValueType.STRING); terms.field("foo") .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")) .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field")); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java index 6c369c3c94f24..379a18a329a93 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java @@ -448,7 +448,7 @@ public void testUnsupportedMultiBucket() throws IOException { .must(QueryBuilders.termQuery("field", "foo")) .should(QueryBuilders.termQuery("field", "bar")); SignificantTermsAggregationBuilder builder = new SignificantTermsAggregationBuilder( - "test", ValueType.STRING) + "test") .field("field") .backgroundFilter(filter); @@ -1015,9 +1015,11 @@ public void testMetric() throws IOException { public void testUnsupportedMetric() throws IOException { - AggregationBuilder nonRollup = new CardinalityAggregationBuilder("test_metric", ValueType.LONG).field("foo"); + AggregationBuilder nonRollup = new CardinalityAggregationBuilder("test_metric").userValueTypeHint(ValueType.LONG) + .field("foo"); String fieldName = "foo.max." + RollupField.VALUE; - AggregationBuilder rollup = new CardinalityAggregationBuilder("test_metric", ValueType.LONG).field(fieldName); + AggregationBuilder rollup = new CardinalityAggregationBuilder("test_metric").userValueTypeHint(ValueType.LONG) + .field(fieldName); NumberFieldMapper.Builder nrValueMapper = new NumberFieldMapper.Builder("foo", NumberFieldMapper.NumberType.LONG); @@ -1049,10 +1051,10 @@ public void testUnsupportedMetric() throws IOException { } public void testStringTerms() throws IOException { - TermsAggregationBuilder nonRollupTerms = new TermsAggregationBuilder("terms", ValueType.STRING) + TermsAggregationBuilder nonRollupTerms = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING) .field("stringField"); - TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms", ValueType.STRING) + TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING) .field("stringfield.terms." + RollupField.VALUE) .subAggregation(new SumAggregationBuilder("terms." + RollupField.COUNT_FIELD) .field("stringfield.terms." + RollupField.COUNT_FIELD)); @@ -1090,10 +1092,10 @@ public void testStringTerms() throws IOException { } public void testStringTermsNullValue() throws IOException { - TermsAggregationBuilder nonRollupTerms = new TermsAggregationBuilder("terms", ValueType.STRING) + TermsAggregationBuilder nonRollupTerms = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING) .field("stringField"); - TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms", ValueType.STRING) + TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.STRING) .field("stringfield.terms." + RollupField.VALUE) .subAggregation(new SumAggregationBuilder("terms." + RollupField.COUNT_FIELD) .field("stringfield.terms." + RollupField.COUNT_FIELD)); @@ -1138,10 +1140,10 @@ public void testStringTermsNullValue() throws IOException { } public void testLongTerms() throws IOException { - TermsAggregationBuilder nonRollupTerms = new TermsAggregationBuilder("terms", ValueType.LONG) + TermsAggregationBuilder nonRollupTerms = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.LONG) .field("longField"); - TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms", ValueType.LONG) + TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.LONG) .field("longfield.terms." + RollupField.VALUE) .subAggregation(new SumAggregationBuilder("terms." + RollupField.COUNT_FIELD) .field("longfield.terms." + RollupField.COUNT_FIELD)); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index baabbfd8d3e94..b0c22b9db910e 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -91,7 +91,7 @@ private void setup() { settings = createIndexSettings(); queryShardContext = new QueryShardContext(0, settings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - null, null, null, null, () -> 0L, null, null, () -> true); + null, null, null, null, () -> 0L, null, null, () -> true, null); } public void testSimpleDateHisto() throws Exception { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Aggregations.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Aggregations.java index eb2289088eafc..4129583f33457 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Aggregations.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Aggregations.java @@ -183,7 +183,7 @@ public static Tuple, Map> getAggregationInpu } if (agg instanceof ValuesSourceAggregationBuilder) { - ValuesSourceAggregationBuilder valueSourceAggregation = (ValuesSourceAggregationBuilder) agg; + ValuesSourceAggregationBuilder valueSourceAggregation = (ValuesSourceAggregationBuilder) agg; return new Tuple<>( Collections.singletonMap(valueSourceAggregation.getName(), valueSourceAggregation.field()), Collections.singletonMap(agg.getName(), agg.getType()) diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 568e4f502dbe0..ffb3a81aeaf67 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -59,6 +59,8 @@ import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -76,15 +78,15 @@ public class WildcardFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "wildcard"; - public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10; - public static final int NGRAM_SIZE = 3; + public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10; + public static final int NGRAM_SIZE = 3; static final NamedAnalyzer WILDCARD_ANALYZER = new NamedAnalyzer("_wildcard", AnalyzerScope.GLOBAL, new Analyzer() { @Override public TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new NGramTokenizer(NGRAM_SIZE, NGRAM_SIZE); return new TokenStreamComponents(tokenizer); } - }); + }); public static class Defaults { public static final MappedFieldType FIELD_TYPE = new WildcardFieldType(); @@ -98,14 +100,14 @@ public static class Defaults { FIELD_TYPE.setOmitNorms(true); FIELD_TYPE.freeze(); } - public static final int IGNORE_ABOVE = Integer.MAX_VALUE; + public static final int IGNORE_ABOVE = Integer.MAX_VALUE; } public static class Builder extends FieldMapper.Builder { protected int ignoreAbove = Defaults.IGNORE_ABOVE; private IndexAnalyzers indexAnalyzers; private String normalizerName; - + public Builder(String name) { super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); @@ -115,7 +117,7 @@ public Builder(String name) { @Override public Builder docValues(boolean docValues) { if (docValues == false) { - throw new MapperParsingException("The field [" + name + "] cannot have doc values = false"); + throw new MapperParsingException("The field [" + name + "] cannot have doc values = false"); } return this; } @@ -131,64 +133,64 @@ public Builder indexOptions(IndexOptions indexOptions) { @Override public Builder store(boolean store) { if (store) { - throw new MapperParsingException("The field [" + name + "] cannot have store = true"); + throw new MapperParsingException("The field [" + name + "] cannot have store = true"); } return this; } @Override public Builder similarity(SimilarityProvider similarity) { - throw new MapperParsingException("The field [" + name + "] cannot have custom similarities"); + throw new MapperParsingException("The field [" + name + "] cannot have custom similarities"); } @Override public Builder index(boolean index) { if (index == false) { - throw new MapperParsingException("The field [" + name + "] cannot have index = false"); + throw new MapperParsingException("The field [" + name + "] cannot have index = false"); } return this; } - + public Builder ignoreAbove(int ignoreAbove) { if (ignoreAbove < 0) { throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove); } this.ignoreAbove = ignoreAbove; return this; - } - - + } + + @Override protected void setupFieldType(BuilderContext context) { super.setupFieldType(context); fieldType().setHasDocValues(true); fieldType().setTokenized(false); - fieldType().setIndexOptions(IndexOptions.DOCS); + fieldType().setIndexOptions(IndexOptions.DOCS); } @Override public WildcardFieldType fieldType() { return (WildcardFieldType) super.fieldType(); } - + public Builder normalizer(IndexAnalyzers indexAnalyzers, String name) { this.indexAnalyzers = indexAnalyzers; this.normalizerName = name; return builder; - } + } @Override public WildcardFieldMapper build(BuilderContext context) { - setupFieldType(context); + setupFieldType(context); if (normalizerName != null) { NamedAnalyzer normalizer = indexAnalyzers.getNormalizer(normalizerName); if (normalizer == null) { throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]"); } fieldType().setNormalizer(normalizer); - } + } return new WildcardFieldMapper( - name, fieldType, defaultFieldType, ignoreAbove, + name, fieldType, defaultFieldType, ignoreAbove, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } } @@ -199,7 +201,7 @@ public static class TypeParser implements Mapper.TypeParser { throws MapperParsingException { WildcardFieldMapper.Builder builder = new WildcardFieldMapper.Builder(name); parseField(builder, name, node, parserContext); - + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String propName = entry.getKey(); @@ -213,21 +215,21 @@ public static class TypeParser implements Mapper.TypeParser { } iterator.remove(); } - } - + } + return builder; } } - - public static final char TOKEN_START_OR_END_CHAR = 0; - + + public static final char TOKEN_START_OR_END_CHAR = 0; + public static final class WildcardFieldType extends MappedFieldType { - + private NamedAnalyzer normalizer = null; - public WildcardFieldType() { + public WildcardFieldType() { setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); - setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); + setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); } protected WildcardFieldType(WildcardFieldType ref) { @@ -239,7 +241,7 @@ public WildcardFieldType clone() { WildcardFieldType result = new WildcardFieldType(this); return result; } - + @Override public boolean equals(Object o) { @@ -253,8 +255,8 @@ public boolean equals(Object o) { @Override public int hashCode() { return 31 * super.hashCode() + Objects.hash(normalizer); - } - + } + private NamedAnalyzer normalizer() { return normalizer; } @@ -262,7 +264,7 @@ private NamedAnalyzer normalizer() { public void setNormalizer(NamedAnalyzer normalizer) { checkIfFrozen(); this.normalizer = normalizer; - } + } @Override public void checkCompatibility(MappedFieldType otherFT, List conflicts) { @@ -272,22 +274,22 @@ public void checkCompatibility(MappedFieldType otherFT, List conflicts) conflicts.add("mapper [" + name() + "] has different [normalizer]"); } } - + // Holds parsed information about the wildcard pattern static class PatternStructure { - boolean openStart, openEnd, hasSymbols; + boolean openStart, openEnd, hasSymbols; int lastGap =0; int wildcardCharCount, wildcardStringCount; String[] fragments; Integer [] precedingGapSizes; final String pattern; - + @SuppressWarnings("fallthrough") // Intentionally uses fallthrough mirroring implementation in Lucene's WildcardQuery PatternStructure (String wildcardText) { this.pattern = wildcardText; ArrayList fragmentList = new ArrayList<>(); ArrayList precedingGapSizeList = new ArrayList<>(); - StringBuilder sb = new StringBuilder(); + StringBuilder sb = new StringBuilder(); for (int i = 0; i < wildcardText.length();) { final int c = wildcardText.codePointAt(i); int length = Character.charCount(c); @@ -296,10 +298,10 @@ static class PatternStructure { if (i == 0) { openStart = true; } - openEnd = true; + openEnd = true; hasSymbols = true; wildcardStringCount++; - + if (sb.length() > 0) { precedingGapSizeList.add(lastGap); fragmentList.add(sb.toString()); @@ -317,10 +319,10 @@ static class PatternStructure { if (sb.length() > 0) { precedingGapSizeList.add(lastGap); fragmentList.add(sb.toString()); - sb = new StringBuilder(); + sb = new StringBuilder(); lastGap = 0; } - + if (lastGap != Integer.MAX_VALUE) { lastGap++; } @@ -347,19 +349,19 @@ static class PatternStructure { } fragments = fragmentList.toArray(new String[0]); precedingGapSizes = precedingGapSizeList.toArray(new Integer[0]); - + } - + public boolean needsVerification() { // Return true if term queries are not enough evidence if (fragments.length == 1 && wildcardCharCount == 0) { - // The one case where we don't need verification is when + // The one case where we don't need verification is when // we have a single fragment and no ? characters return false; } return true; } - + // Returns number of positions for last gap (Integer.MAX means unlimited gap) public int getPrecedingGapSize(int fragmentNum) { return precedingGapSizes[fragmentNum]; @@ -379,17 +381,17 @@ public boolean equals(Object obj) { PatternStructure other = (PatternStructure) obj; return pattern.equals(other.pattern); } - - + + } - + @Override public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) { if (normalizer != null) { wildcardPattern = StringFieldType.normalizeWildcardPattern(name(), wildcardPattern, normalizer); } - PatternStructure patternStructure = new PatternStructure(wildcardPattern); + PatternStructure patternStructure = new PatternStructure(wildcardPattern); ArrayList tokens = new ArrayList<>(); for (int i = 0; i < patternStructure.fragments.length; i++) { @@ -398,7 +400,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh if (fLength == 0) { continue; } - + // Add any start/end of string character if (i == 0 && patternStructure.openStart == false) { // Start-of-string anchored (is not a leading wildcard) @@ -411,7 +413,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh if (fragment.codePointCount(0, fragment.length()) <= NGRAM_SIZE) { tokens.add(fragment); } else { - // Break fragment into multiple Ngrams + // Break fragment into multiple Ngrams TokenStream tokenizer = WILDCARD_ANALYZER.tokenStream(name(), fragment); CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class); String lastUnusedToken = null; @@ -425,7 +427,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh if (takeThis) { tokens.add(tokenValue); } else { - lastUnusedToken = tokenValue; + lastUnusedToken = tokenValue; } // alternate takeThis = !takeThis; @@ -445,7 +447,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh if (patternStructure.isMatchAll()) { return new MatchAllDocsQuery(); - } + } BooleanQuery approximation = createApproximationQuery(tokens); if (approximation.clauses().size() > 1 || patternStructure.needsVerification()) { BooleanQuery.Builder verifyingBuilder = new BooleanQuery.Builder(); @@ -455,7 +457,7 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh return verifyingBuilder.build(); } return approximation; - } + } private BooleanQuery createApproximationQuery(ArrayList tokens) { BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder(); @@ -473,7 +475,7 @@ private BooleanQuery createApproximationQuery(ArrayList tokens) { // TODO we can be smarter about pruning here. e.g. // * Avoid wildcard queries if there are sufficient numbers of other terms that are full 3grams that are cheaper term queries // * We can select terms on their scarcity rather than even spreads across the search string. - + return bqBuilder.build(); } @@ -504,11 +506,11 @@ public Query existsQuery(QueryShardContext context) { public Query termQuery(Object value, QueryShardContext context) { return wildcardQuery(BytesRefs.toString(value), MultiTermQuery.CONSTANT_SCORE_REWRITE, context); } - + @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { return wildcardQuery(value + "*", method, context); - } + } @Override public Query termsQuery(List values, QueryShardContext context) { @@ -517,8 +519,8 @@ public Query termsQuery(List values, QueryShardContext context) { bq.add(termQuery(value, context), Occur.SHOULD); } return new ConstantScoreQuery(bq.build()); - } - + } + @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); @@ -531,8 +533,12 @@ public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fiel }}; } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } - String normalize(String value) throws IOException { + String normalize(String value) throws IOException { if (normalizer != null) { try (TokenStream ts = normalizer.tokenStream(name(), value)) { final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); @@ -553,10 +559,10 @@ String normalize(String value) throws IOException { } } return value; - } - + } + } - + static class WildcardBytesBinaryDVIndexFieldData extends BytesBinaryDVIndexFieldData{ WildcardBytesBinaryDVIndexFieldData(Index index, String fieldName) { @@ -569,7 +575,7 @@ public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested sortMode, nested); return new SortField(getFieldName(), source, reverse); } - + } private int ignoreAbove; @@ -579,9 +585,9 @@ private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, Mapped super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); this.ignoreAbove = ignoreAbove; assert fieldType.indexOptions() == IndexOptions.DOCS; - + ngramFieldType = fieldType.clone(); - ngramFieldType.setTokenized(true); + ngramFieldType.setTokenized(true); ngramFieldType.freeze(); } @@ -590,8 +596,8 @@ private WildcardFieldMapper(String simpleName, MappedFieldType fieldType, Mapped // pkg-private for testing int ignoreAbove() { return ignoreAbove; - } - + } + @Override protected WildcardFieldMapper clone() { return (WildcardFieldMapper) super.clone(); @@ -612,9 +618,9 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, builder.field("normalizer", fieldType().normalizer().name()); } else if (includeDefaults) { builder.nullField("normalizer"); - } + } } - + @Override protected void parseCreateField(ParseContext context, List fields) throws IOException { final String value; @@ -629,13 +635,13 @@ protected void parseCreateField(ParseContext context, List field } } ParseContext.Document parseDoc = context.doc(); - - createFields(value, parseDoc, fields); - } - + + createFields(value, parseDoc, fields); + } + // For internal use by Lucene only - used to define ngram index final MappedFieldType ngramFieldType; - + void createFields(String value, Document parseDoc, Listfields) throws IOException { if (value == null || value.length() > ignoreAbove) { return; @@ -644,25 +650,25 @@ void createFields(String value, Document parseDoc, Listfields) t String ngramValue = TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR; Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType); fields.add(ngramField); - + CustomBinaryDocValuesField dvField = (CustomBinaryDocValuesField) parseDoc.getByKey(fieldType().name()); if (dvField == null) { dvField = new CustomBinaryDocValuesField(fieldType().name(), value.getBytes(StandardCharsets.UTF_8)); parseDoc.addWithKey(fieldType().name(), dvField); } else { dvField.add(value.getBytes(StandardCharsets.UTF_8)); - } + } } @Override protected String contentType() { return CONTENT_TYPE; } - + @Override protected void doMerge(Mapper mergeWith) { super.doMerge(mergeWith); this.ignoreAbove = ((WildcardFieldMapper) mergeWith).ignoreAbove; - } + } } diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index f8c8ddc7f5eef..9d9f7eb4d1b9d 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -68,21 +68,21 @@ public class WildcardFieldMapperTests extends ESTestCase { @Before public void setUp() throws Exception { Builder builder = new WildcardFieldMapper.Builder(WILDCARD_FIELD_NAME); - builder.ignoreAbove(MAX_FIELD_LENGTH); + builder.ignoreAbove(MAX_FIELD_LENGTH); wildcardFieldType = builder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0))); - - + + org.elasticsearch.index.mapper.KeywordFieldMapper.Builder kwBuilder = new KeywordFieldMapper.Builder(KEYWORD_FIELD_NAME); - keywordFieldType = kwBuilder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0))); + keywordFieldType = kwBuilder.build(new Mapper.BuilderContext(createIndexSettings().getSettings(), new ContentPath(0))); super.setUp(); } - + public void testIllegalDocValuesArgument() { Builder ft = new WildcardFieldMapper.Builder("test"); MapperParsingException e = expectThrows(MapperParsingException.class, () -> ft.docValues(false)); assertEquals("The field [test] cannot have doc values = false", e.getMessage()); - } + } public void testIllegalIndexedArgument() { Builder ft = new WildcardFieldMapper.Builder("test"); @@ -90,19 +90,19 @@ public void testIllegalIndexedArgument() { () -> ft.index(false)); assertEquals("The field [test] cannot have index = false", e.getMessage()); } - + public void testTooBigKeywordField() throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER); iwc.setMergePolicy(newTieredMergePolicy(random())); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); - + // Create a string that is too large and will not be indexed String docContent = randomABString(MAX_FIELD_LENGTH + 1); Document doc = new Document(); - ParseContext.Document parseDoc = new ParseContext.Document(); + ParseContext.Document parseDoc = new ParseContext.Document(); addFields(parseDoc, doc, docContent); - indexDoc(parseDoc, doc, iw); + indexDoc(parseDoc, doc, iw); iw.forceMerge(1); DirectoryReader reader = iw.getReader(); @@ -112,24 +112,24 @@ public void testTooBigKeywordField() throws IOException { Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery("*a*", null, null); TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER); assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L)); - + reader.close(); - dir.close(); + dir.close(); } - + //Test long query strings don't cause exceptions public void testTooBigQueryField() throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER); iwc.setMergePolicy(newTieredMergePolicy(random())); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); - + // Create a string that is too large and will not be indexed String docContent = randomABString(10); Document doc = new Document(); - ParseContext.Document parseDoc = new ParseContext.Document(); + ParseContext.Document parseDoc = new ParseContext.Document(); addFields(parseDoc, doc, docContent); - indexDoc(parseDoc, doc, iw); + indexDoc(parseDoc, doc, iw); iw.forceMerge(1); DirectoryReader reader = iw.getReader(); @@ -140,12 +140,12 @@ public void testTooBigQueryField() throws IOException { Query wildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery(queryString, null, null); TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER); assertThat(wildcardFieldTopDocs.totalHits.value, equalTo(0L)); - + reader.close(); - dir.close(); - } - - + dir.close(); + } + + public void testSearchResultsVersusKeywordField() throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER); @@ -156,22 +156,22 @@ public void testSearchResultsVersusKeywordField() throws IOException { HashSet values = new HashSet<>(); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - ParseContext.Document parseDoc = new ParseContext.Document(); + ParseContext.Document parseDoc = new ParseContext.Document(); String docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1)); - if (values.contains(docContent) == false) { + if (values.contains(docContent) == false) { addFields(parseDoc, doc, docContent); values.add(docContent); } // Occasionally add a multi-value field if (randomBoolean()) { docContent = randomABString(1 + randomInt(MAX_FIELD_LENGTH - 1)); - if (values.contains(docContent) == false) { + if (values.contains(docContent) == false) { addFields(parseDoc, doc, docContent); values.add(docContent); - } + } } indexDoc(parseDoc, doc, iw); - + } iw.forceMerge(1); @@ -200,38 +200,38 @@ public void testSearchResultsVersusKeywordField() throws IOException { } assertThat(expectedDocs.size(), equalTo(0)); } - - + + //Test keyword and wildcard sort operations are also equivalent QueryShardContext shardContextMock = createMockShardContext(); - + FieldSortBuilder wildcardSortBuilder = new FieldSortBuilder(WILDCARD_FIELD_NAME); - SortField wildcardSortField = wildcardSortBuilder.build(shardContextMock).field; + SortField wildcardSortField = wildcardSortBuilder.build(shardContextMock).field; ScoreDoc[] wildcardHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(wildcardSortField)).scoreDocs; FieldSortBuilder keywordSortBuilder = new FieldSortBuilder(KEYWORD_FIELD_NAME); - SortField keywordSortField = keywordSortBuilder.build(shardContextMock).field; + SortField keywordSortField = keywordSortBuilder.build(shardContextMock).field; ScoreDoc[] keywordHits = searcher.search(new MatchAllDocsQuery(), numDocs, new Sort(keywordSortField)).scoreDocs; - + assertThat(wildcardHits.length, equalTo(keywordHits.length)); for (int i = 0; i < wildcardHits.length; i++) { - assertThat(wildcardHits[i].doc, equalTo(keywordHits[i].doc)); + assertThat(wildcardHits[i].doc, equalTo(keywordHits[i].doc)); } - + reader.close(); dir.close(); } - - + + protected MappedFieldType provideMappedFieldType(String name) { if (name.equals(WILDCARD_FIELD_NAME)) { - return wildcardFieldType.fieldType(); + return wildcardFieldType.fieldType(); } else { return keywordFieldType.fieldType(); } - } - + } + protected final QueryShardContext createMockShardContext() { Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_"); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, @@ -243,15 +243,15 @@ protected final QueryShardContext createMockShardContext() { }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, null, xContentRegistry(), null, null, null, - () -> randomNonNegativeLong(), null, null, () -> true) { + () -> randomNonNegativeLong(), null, null, () -> true, null) { @Override public MappedFieldType fieldMapper(String name) { return provideMappedFieldType(name); } }; - } - + } + private void addFields(ParseContext.Document parseDoc, Document doc, String docContent) throws IOException { ArrayList fields = new ArrayList<>(); wildcardFieldType.createFields(docContent, parseDoc, fields); @@ -260,7 +260,7 @@ private void addFields(ParseContext.Document parseDoc, Document doc, String docC doc.add(indexableField); } // Add keyword fields too - doc.add(new SortedSetDocValuesField(KEYWORD_FIELD_NAME, new BytesRef(docContent))); + doc.add(new SortedSetDocValuesField(KEYWORD_FIELD_NAME, new BytesRef(docContent))); doc.add(new StringField(KEYWORD_FIELD_NAME, docContent, Field.Store.YES)); }