diff --git a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java index e91e0915bca91..d2f2aec3c76dd 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java @@ -114,6 +114,13 @@ default List> getQueries() { default List getAggregations() { return emptyList(); } + /** + * Allows plugins to register new aggregations using aggregation names that are already defined + * in Core, as long as the new aggregations target different ValuesSourceTypes + */ + default List> getBareAggregatorRegistrar() { + return emptyList(); + } /** * The new {@link PipelineAggregator}s added by this plugin. */ diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 270732ffa76d4..5acfdb38e7724 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -458,6 +458,9 @@ private void registerAggregations(List plugins) { registerAggregation((new AggregationSpec(CompositeAggregationBuilder.NAME, CompositeAggregationBuilder::new, CompositeAggregationBuilder.PARSER).addResultReader(InternalComposite::new))); registerFromPlugin(plugins, SearchPlugin::getAggregations, this::registerAggregation); + + // after aggs have been registered, see if there are any new VSTypes that need to be linked to core fields + registerFromPlugin(plugins, SearchPlugin::getBareAggregatorRegistrar, this::registerBareAggregatorRegistrar); } private void registerAggregation(AggregationSpec spec) { @@ -478,6 +481,12 @@ private void registerAggregation(AggregationSpec spec) { } } + private void registerBareAggregatorRegistrar(Consumer registrar) { + if (registrar != null) { + registrar.accept(this.valuesSourceRegistry); + } + } + private void registerPipelineAggregations(List plugins) { registerPipelineAggregation(new PipelineAggregationSpec( DerivativePipelineAggregationBuilder.NAME, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java index 5d9e616a39718..546db9d447485 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java @@ -26,8 +26,6 @@ import org.elasticsearch.common.util.ArrayUtils; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ObjectArray; -import org.elasticsearch.index.fielddata.HistogramValue; -import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; @@ -78,18 +76,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - if (valuesSource instanceof ValuesSource.Histogram) { - final HistogramValues values = ((ValuesSource.Histogram)valuesSource).getHistogramValues(ctx); - return collectHistogramValues(values, bigArrays, sub); - } else { - final SortedNumericDoubleValues values = ((ValuesSource.Numeric)valuesSource).doubleValues(ctx); - return collectNumeric(values, bigArrays, sub); - } - - } - private LeafBucketCollector collectNumeric(final SortedNumericDoubleValues values, - final BigArrays bigArrays, final LeafBucketCollector sub) { + final SortedNumericDoubleValues values = ((ValuesSource.Numeric)valuesSource).doubleValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -102,22 +90,7 @@ public void collect(int doc, long bucket) throws IOException { } } }; - } - private LeafBucketCollector collectHistogramValues(final HistogramValues values, - final BigArrays bigArrays, final LeafBucketCollector sub) { - return new LeafBucketCollectorBase(sub, values) { - @Override - public void collect(int doc, long bucket) throws IOException { - DoubleHistogram state = getExistingOrNewHistogram(bigArrays, bucket); - if (values.advanceExact(doc)) { - final HistogramValue sketch = values.histogram(); - while (sketch.next()) { - state.recordValueWithCount(sketch.value(), sketch.count()); - } - } - } - }; } private DoubleHistogram getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) { @@ -125,12 +98,13 @@ private DoubleHistogram getExistingOrNewHistogram(final BigArrays bigArrays, lon DoubleHistogram state = states.get(bucket); if (state == null) { state = new DoubleHistogram(numberOfSignificantValueDigits); - // Set the histogram to autosize so it can resize itself as - // the data range increases. Resize operations should be - // rare as the histogram buckets are exponential (on the top - // level). In the future we could expose the range as an - // option on the request so the histogram can be fixed at - // initialisation and doesn't need resizing. + /* Set the histogram to autosize so it can resize itself as + the data range increases. Resize operations should be + rare as the histogram buckets are exponential (on the top + level). In the future we could expose the range as an + option on the request so the histogram can be fixed at + initialisation and doesn't need resizing. + */ state.setAutoResize(true); states.set(bucket, state); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java index 1b78db480068e..4823bd805fb17 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java @@ -25,8 +25,6 @@ import org.elasticsearch.common.util.ArrayUtils; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ObjectArray; -import org.elasticsearch.index.fielddata.HistogramValue; -import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; @@ -77,18 +75,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - if (valuesSource instanceof ValuesSource.Histogram) { - final HistogramValues values = ((ValuesSource.Histogram)valuesSource).getHistogramValues(ctx); - return collectHistogramValues(values, bigArrays, sub); - } else { - final SortedNumericDoubleValues values = ((ValuesSource.Numeric)valuesSource).doubleValues(ctx); - return collectNumeric(values, bigArrays, sub); - } - - } - - private LeafBucketCollector collectNumeric(final SortedNumericDoubleValues values, - final BigArrays bigArrays, final LeafBucketCollector sub) { + final SortedNumericDoubleValues values = ((ValuesSource.Numeric)valuesSource).doubleValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -103,22 +90,6 @@ public void collect(int doc, long bucket) throws IOException { }; } - private LeafBucketCollector collectHistogramValues(final HistogramValues values, - final BigArrays bigArrays, final LeafBucketCollector sub) { - return new LeafBucketCollectorBase(sub, values) { - @Override - public void collect(int doc, long bucket) throws IOException { - TDigestState state = getExistingOrNewHistogram(bigArrays, bucket); - if (values.advanceExact(doc)) { - final HistogramValue sketch = values.histogram(); - while(sketch.next()) { - state.add(sketch.value(), sketch.count()); - } - } - } - }; - } - private TDigestState getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) { states = bigArrays.grow(states, bucket + 1); TDigestState state = states.get(bucket); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java index bfe483d0e3c47..2d684e9b8e637 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java @@ -31,8 +31,8 @@ public class InternalHDRPercentileRanks extends AbstractInternalHDRPercentiles implements PercentileRanks { public static final String NAME = "hdr_percentile_ranks"; - InternalHDRPercentileRanks(String name, double[] cdfValues, DoubleHistogram state, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, Map metaData) { + public InternalHDRPercentileRanks(String name, double[] cdfValues, DoubleHistogram state, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) { super(name, cdfValues, state, keyed, formatter, pipelineAggregators, metaData); } @@ -74,7 +74,7 @@ protected AbstractInternalHDRPercentiles createReduced(String name, double[] key return new InternalHDRPercentileRanks(name, keys, merged, keyed, format, pipelineAggregators, metaData); } - static double percentileRank(DoubleHistogram state, double value) { + public static double percentileRank(DoubleHistogram state, double value) { if (state.getTotalCount() == 0) { return Double.NaN; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java index 5a62de8a964ec..6f47169e7ed6c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java @@ -31,8 +31,8 @@ public class InternalHDRPercentiles extends AbstractInternalHDRPercentiles implements Percentiles { public static final String NAME = "hdr_percentiles"; - InternalHDRPercentiles(String name, double[] percents, DoubleHistogram state, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, Map metaData) { + public InternalHDRPercentiles(String name, double[] percents, DoubleHistogram state, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) { super(name, percents, state, keyed, formatter, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java index aa82ac5ba6add..69cc407efa032 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentileRanks.java @@ -30,8 +30,8 @@ public class InternalTDigestPercentileRanks extends AbstractInternalTDigestPercentiles implements PercentileRanks { public static final String NAME = "tdigest_percentile_ranks"; - InternalTDigestPercentileRanks(String name, double[] cdfValues, TDigestState state, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, Map metaData) { + public InternalTDigestPercentileRanks(String name, double[] cdfValues, TDigestState state, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) { super(name, cdfValues, state, keyed, formatter, pipelineAggregators, metaData); } @@ -73,7 +73,7 @@ protected AbstractInternalTDigestPercentiles createReduced(String name, double[] return new InternalTDigestPercentileRanks(name, keys, merged, keyed, format, pipelineAggregators, metaData); } - static double percentileRank(TDigestState state, double value) { + public static double percentileRank(TDigestState state, double value) { double percentileRank = state.cdf(value); if (percentileRank < 0) { percentileRank = 0; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentiles.java index 28f1230bec713..6c858d7a1ec48 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentiles.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentiles.java @@ -30,8 +30,8 @@ public class InternalTDigestPercentiles extends AbstractInternalTDigestPercentiles implements Percentiles { public static final String NAME = "tdigest_percentiles"; - InternalTDigestPercentiles(String name, double[] percents, TDigestState state, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, Map metaData) { + public InternalTDigestPercentiles(String name, double[] percents, TDigestState state, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) { super(name, percents, state, keyed, formatter, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java index a2c827bff373f..39080f5d6f62e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java @@ -46,7 +46,7 @@ class PercentileRanksAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { valuesSourceRegistry.register(PercentileRanksAggregationBuilder.NAME, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.HISTOGRAM, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), new PercentilesAggregatorSupplier() { @Override public Aggregator build(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java index f8ed217c6d2ba..8b3112613e7b7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java @@ -50,7 +50,7 @@ class PercentilesAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { valuesSourceRegistry.register(PercentilesAggregationBuilder.NAME, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.HISTOGRAM, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), new PercentilesAggregatorSupplier() { @Override public Aggregator build(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesConfig.java index af7f9ed5ec412..67af2c18102b1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesConfig.java @@ -69,10 +69,10 @@ public PercentilesMethod getMethod() { return method; } - abstract Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, - double[] values, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, - Map metaData) throws IOException; + public abstract Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] values, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException; abstract Aggregator createPercentileRanksAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, double[] values, boolean keyed, @@ -102,11 +102,11 @@ public static class TDigest extends PercentilesConfig { static final double DEFAULT_COMPRESSION = 100.0; private double compression; - TDigest() { + public TDigest() { this(DEFAULT_COMPRESSION); } - TDigest(double compression) { + public TDigest(double compression) { super(PercentilesMethod.TDIGEST); setCompression(compression); } @@ -128,10 +128,10 @@ public double getCompression() { } @Override - Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, - double[] values, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, - Map metaData) throws IOException { + public Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] values, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException { return new TDigestPercentilesAggregator(name, valuesSource, context, parent, values, compression, keyed, formatter, pipelineAggregators, metaData); } @@ -179,11 +179,11 @@ public static class Hdr extends PercentilesConfig { static final int DEFAULT_NUMBER_SIG_FIGS = 3; private int numberOfSignificantValueDigits; - Hdr() { + public Hdr() { this(DEFAULT_NUMBER_SIG_FIGS); } - Hdr(int numberOfSignificantValueDigits) { + public Hdr(int numberOfSignificantValueDigits) { super(PercentilesMethod.HDR); setNumberOfSignificantValueDigits(numberOfSignificantValueDigits); } @@ -204,10 +204,10 @@ public int getNumberOfSignificantValueDigits() { } @Override - Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, - double[] values, boolean keyed, DocValueFormat formatter, - List pipelineAggregators, - Map metaData) throws IOException { + public Aggregator createPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] values, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException { return new HDRPercentilesAggregator(name, valuesSource, context, parent, values, numberOfSignificantValueDigits, keyed, formatter, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index a433c53ca9c93..d7f38cfa16e08 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.mapper.DateFieldMapper; @@ -177,34 +176,6 @@ public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, throw new IllegalArgumentException("Can't apply missing values on a " + valuesSource.getClass()); } }, - HISTOGRAM(EquivalenceType.HISTOGRAM) { - @Override - public ValuesSource getEmpty() { - // TODO: Is this the correct exception type here? - throw new IllegalArgumentException("Can't deal with unmapped ValuesSource type " + this.value()); - } - - @Override - public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { - throw new AggregationExecutionException("value source of type [" + this.value() + "] is not supported by scripts"); - } - - @Override - public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { - final IndexFieldData indexFieldData = fieldContext.indexFieldData(); - - if (!(indexFieldData instanceof IndexHistogramFieldData)) { - throw new IllegalArgumentException("Expected histogram type on field [" + fieldContext.field() + - "], but got [" + fieldContext.fieldType().typeName() + "]"); - } - return new ValuesSource.Histogram.Fielddata((IndexHistogramFieldData) indexFieldData); - } - - @Override - public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { - throw new IllegalArgumentException("Can't apply missing values on a " + valuesSource.getClass()); - } - }, // TODO: Ordinal Numbering sync with types from master IP(EquivalenceType.STRING) { @Override @@ -292,7 +263,7 @@ public DocValueFormat getFormatter(String format, ZoneId tz) { ; enum EquivalenceType { - STRING, NUMBER, GEO, RANGE, HISTOGRAM; + STRING, NUMBER, GEO, RANGE; } EquivalenceType equivalenceType; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index 3f77744b5a56e..3f29ce627092e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -33,10 +33,8 @@ import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.DocValueBits; -import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.fielddata.MultiGeoPointValues; @@ -565,39 +563,4 @@ public org.elasticsearch.index.fielddata.MultiGeoPointValues geoPointValues(Leaf } } } - - public abstract static class Histogram extends ValuesSource { - - public abstract HistogramValues getHistogramValues(LeafReaderContext context) throws IOException; - - public static class Fielddata extends Histogram { - - protected final IndexHistogramFieldData indexFieldData; - - public Fielddata(IndexHistogramFieldData indexFieldData) { - this.indexFieldData = indexFieldData; - } - - @Override - public SortedBinaryDocValues bytesValues(LeafReaderContext context) { - return indexFieldData.load(context).getBytesValues(); - } - - @Override - public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { - HistogramValues values = getHistogramValues(context); - return new DocValueBits() { - @Override - public boolean advanceExact(int doc) throws IOException { - return values.advanceExact(doc); - } - }; - } - - public HistogramValues getHistogramValues(LeafReaderContext context) throws IOException { - return indexFieldData.load(context).getHistogramValues(); - } - } - } - } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index 96bdda5dfb830..6cd62a0d2a622 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; @@ -200,8 +199,6 @@ private static ValuesSourceType getLegacyMapping( return CoreValuesSourceType.GEOPOINT; } else if (fieldType instanceof RangeFieldMapper.RangeFieldType) { return CoreValuesSourceType.RANGE; - } else if (indexFieldData instanceof IndexHistogramFieldData) { - return CoreValuesSourceType.HISTOGRAM; } else { if (userValueTypeHint == null) { return defaultValuesSourceType; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java index 91b86fb50aca8..5356821c5f58f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java @@ -20,7 +20,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; @@ -160,8 +159,6 @@ && findMatchingSuppier(valuesSourceType, aggregatorRegistry.get(aggregationName) return CoreValuesSourceType.GEOPOINT; } else if (fieldType instanceof RangeFieldMapper.RangeFieldType) { return CoreValuesSourceType.RANGE; - } else if (indexFieldData instanceof IndexHistogramFieldData) { - return CoreValuesSourceType.HISTOGRAM; } else { if (valueType == null) { return defaultValuesSourceType; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index c3956121ac625..fe0695a98b0e4 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -451,7 +451,6 @@ protected List getSupportedValuesSourceTypes() { CoreValuesSourceType.BYTES, CoreValuesSourceType.GEOPOINT, CoreValuesSourceType.RANGE, - CoreValuesSourceType.HISTOGRAM, CoreValuesSourceType.IP, CoreValuesSourceType.BOOLEAN, CoreValuesSourceType.DATE diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorTests.java index c982cd1c6cc15..a0a0c5a9118a0 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorTests.java @@ -68,8 +68,7 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy protected List getSupportedValuesSourceTypes() { return List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, - CoreValuesSourceType.BOOLEAN, - CoreValuesSourceType.HISTOGRAM); + CoreValuesSourceType.BOOLEAN); } public void testNoDocs() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorTests.java index 766407e86de43..63f8e799fb796 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorTests.java @@ -61,8 +61,7 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy protected List getSupportedValuesSourceTypes() { return List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, - CoreValuesSourceType.BOOLEAN, - CoreValuesSourceType.HISTOGRAM); + CoreValuesSourceType.BOOLEAN); } public void testNoDocs() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java index 8050b2a18e8d3..a9f500bd09154 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java @@ -30,7 +30,6 @@ public void testFromString() { assertThat(CoreValuesSourceType.fromString("bytes"), equalTo(CoreValuesSourceType.BYTES)); assertThat(CoreValuesSourceType.fromString("geopoint"), equalTo(CoreValuesSourceType.GEOPOINT)); assertThat(CoreValuesSourceType.fromString("range"), equalTo(CoreValuesSourceType.RANGE)); - assertThat(CoreValuesSourceType.fromString("histogram"), equalTo(CoreValuesSourceType.HISTOGRAM)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> CoreValuesSourceType.fromString("does_not_exist")); assertThat(e.getMessage(), diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java index 7087a57f2bd44..295cb91ab0bd9 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java @@ -24,11 +24,13 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.analytics.action.AnalyticsInfoTransportAction; import org.elasticsearch.xpack.analytics.action.AnalyticsUsageTransportAction; import org.elasticsearch.xpack.analytics.action.TransportAnalyticsStatsAction; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; import org.elasticsearch.xpack.analytics.boxplot.BoxplotAggregationBuilder; import org.elasticsearch.xpack.analytics.boxplot.InternalBoxplot; import org.elasticsearch.xpack.analytics.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder; @@ -50,6 +52,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import static java.util.Collections.singletonList; @@ -113,6 +116,12 @@ public Map getMappers() { return Collections.singletonMap(HistogramFieldMapper.CONTENT_TYPE, new HistogramFieldMapper.TypeParser()); } + @Override + public List> getBareAggregatorRegistrar() { + return List.of(AnalyticsPercentilesAggregatorFactory::registerPercentilesAggregator, + AnalyticsPercentilesAggregatorFactory::registerPercentileRanksAggregator); + } + @Override public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedHDRPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedHDRPercentilesAggregator.java new file mode 100644 index 0000000000000..86918db07656c --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedHDRPercentilesAggregator.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.HdrHistogram.DoubleHistogram; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ScoreMode; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.util.ArrayUtils; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ObjectArray; +import org.elasticsearch.index.fielddata.HistogramValue; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.analytics.aggregations.support.HistogramValuesSource; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +abstract class AbstractHistoBackedHDRPercentilesAggregator extends NumericMetricsAggregator.MultiValue { + + private static int indexOfKey(double[] keys, double key) { + return ArrayUtils.binarySearch(keys, key, 0.001); + } + + protected final double[] keys; + protected final ValuesSource valuesSource; + protected final DocValueFormat format; + protected ObjectArray states; + protected final int numberOfSignificantValueDigits; + protected final boolean keyed; + + AbstractHistoBackedHDRPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] keys, int numberOfSignificantValueDigits, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) throws IOException { + super(name, context, parent, pipelineAggregators, metaData); + this.valuesSource = valuesSource; + this.keyed = keyed; + this.format = formatter; + this.states = context.bigArrays().newObjectArray(1); + this.keys = keys; + this.numberOfSignificantValueDigits = numberOfSignificantValueDigits; + } + + @Override + public ScoreMode scoreMode() { + return valuesSource != null && valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, + final LeafBucketCollector sub) throws IOException { + if (valuesSource == null) { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + final BigArrays bigArrays = context.bigArrays(); + final HistogramValues values = ((HistogramValuesSource.Histogram)valuesSource).getHistogramValues(ctx); + + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + DoubleHistogram state = getExistingOrNewHistogram(bigArrays, bucket); + if (values.advanceExact(doc)) { + final HistogramValue sketch = values.histogram(); + while (sketch.next()) { + state.recordValueWithCount(sketch.value(), sketch.count()); + } + } + } + }; + } + + private DoubleHistogram getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) { + states = bigArrays.grow(states, bucket + 1); + DoubleHistogram state = states.get(bucket); + if (state == null) { + state = new DoubleHistogram(numberOfSignificantValueDigits); + /* Set the histogram to autosize so it can resize itself as + the data range increases. Resize operations should be + rare as the histogram buckets are exponential (on the top + level). In the future we could expose the range as an + option on the request so the histogram can be fixed at + initialisation and doesn't need resizing. + */ + state.setAutoResize(true); + states.set(bucket, state); + } + return state; + } + + @Override + public boolean hasMetric(String name) { + return indexOfKey(keys, Double.parseDouble(name)) >= 0; + } + + protected DoubleHistogram getState(long bucketOrd) { + if (bucketOrd >= states.size()) { + return null; + } + final DoubleHistogram state = states.get(bucketOrd); + return state; + } + + @Override + protected void doClose() { + Releasables.close(states); + } + +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedTDigestPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedTDigestPercentilesAggregator.java new file mode 100644 index 0000000000000..f4ae77c95b536 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedTDigestPercentilesAggregator.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ScoreMode; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.util.ArrayUtils; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ObjectArray; +import org.elasticsearch.index.fielddata.HistogramValue; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.analytics.aggregations.support.HistogramValuesSource; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +abstract class AbstractHistoBackedTDigestPercentilesAggregator extends NumericMetricsAggregator.MultiValue { + + private static int indexOfKey(double[] keys, double key) { + return ArrayUtils.binarySearch(keys, key, 0.001); + } + + protected final double[] keys; + protected final ValuesSource valuesSource; + protected final DocValueFormat formatter; + protected ObjectArray states; + protected final double compression; + protected final boolean keyed; + + AbstractHistoBackedTDigestPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] keys, double compression, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) throws IOException { + super(name, context, parent, pipelineAggregators, metaData); + this.valuesSource = valuesSource; + this.keyed = keyed; + this.formatter = formatter; + this.states = context.bigArrays().newObjectArray(1); + this.keys = keys; + this.compression = compression; + } + + @Override + public ScoreMode scoreMode() { + return valuesSource != null && valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, + final LeafBucketCollector sub) throws IOException { + if (valuesSource == null) { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + final BigArrays bigArrays = context.bigArrays(); + final HistogramValues values = ((HistogramValuesSource.Histogram)valuesSource).getHistogramValues(ctx); + + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + TDigestState state = getExistingOrNewHistogram(bigArrays, bucket); + if (values.advanceExact(doc)) { + final HistogramValue sketch = values.histogram(); + while(sketch.next()) { + state.add(sketch.value(), sketch.count()); + } + } + } + }; + } + + private TDigestState getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) { + states = bigArrays.grow(states, bucket + 1); + TDigestState state = states.get(bucket); + if (state == null) { + state = new TDigestState(compression); + states.set(bucket, state); + } + return state; + } + + @Override + public boolean hasMetric(String name) { + return indexOfKey(keys, Double.parseDouble(name)) >= 0; + } + + protected TDigestState getState(long bucketOrd) { + if (bucketOrd >= states.size()) { + return null; + } + final TDigestState state = states.get(bucketOrd); + return state; + } + + @Override + protected void doClose() { + Releasables.close(states); + } + +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsPercentilesAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsPercentilesAggregatorFactory.java new file mode 100644 index 0000000000000..2a50482d5f59f --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsPercentilesAggregatorFactory.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesAggregatorSupplier; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; +import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; + +public class AnalyticsPercentilesAggregatorFactory { + public static void registerPercentilesAggregator(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(PercentilesAggregationBuilder.NAME, + AnalyticsValuesSourceType.HISTOGRAM, + (PercentilesAggregatorSupplier) (name, valuesSource, context, parent, percents, percentilesConfig, keyed, + formatter, pipelineAggregators, metaData) -> { + + if (percentilesConfig.getMethod().equals(PercentilesMethod.TDIGEST)) { + double compression = ((PercentilesConfig.TDigest)percentilesConfig).getCompression(); + return new HistoBackedTDigestPercentilesAggregator(name, valuesSource, context, parent, + percents, compression, keyed, formatter, pipelineAggregators, metaData); + + } else if (percentilesConfig.getMethod().equals(PercentilesMethod.HDR)) { + int numSigFig = ((PercentilesConfig.Hdr)percentilesConfig).getNumberOfSignificantValueDigits(); + return new HistoBackedHDRPercentilesAggregator(name, valuesSource, context, parent, + percents, numSigFig, keyed, formatter, pipelineAggregators, metaData); + } + + throw new IllegalArgumentException("Percentiles algorithm: [" + percentilesConfig.getMethod().toString() + "] " + + "is not compatible with Histogram field"); + }); + } + + public static void registerPercentileRanksAggregator(ValuesSourceRegistry valuesSourceRegistry) { + valuesSourceRegistry.register(PercentileRanksAggregationBuilder.NAME, + AnalyticsValuesSourceType.HISTOGRAM, + (PercentilesAggregatorSupplier) (name, valuesSource, context, parent, percents, percentilesConfig, keyed, + formatter, pipelineAggregators, metaData) -> { + + if (percentilesConfig.getMethod().equals(PercentilesMethod.TDIGEST)) { + double compression = ((PercentilesConfig.TDigest)percentilesConfig).getCompression(); + return new HistoBackedTDigestPercentileRanksAggregator(name, valuesSource, context, parent, + percents, compression, keyed, formatter, pipelineAggregators, metaData); + + } else if (percentilesConfig.getMethod().equals(PercentilesMethod.HDR)) { + int numSigFig = ((PercentilesConfig.Hdr)percentilesConfig).getNumberOfSignificantValueDigits(); + return new HistoBackedHDRPercentileRanksAggregator(name, valuesSource, context, parent, + percents, numSigFig, keyed, formatter, pipelineAggregators, metaData); + } + + throw new IllegalArgumentException("Percentiles algorithm: [" + percentilesConfig.getMethod().toString() + "] " + + "is not compatible with Histogram field"); + }); + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java new file mode 100644 index 0000000000000..dc9bca57ec3cd --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.HdrHistogram.DoubleHistogram; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentileRanks; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +class HistoBackedHDRPercentileRanksAggregator extends AbstractHistoBackedHDRPercentilesAggregator { + + HistoBackedHDRPercentileRanksAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] percents, int numberOfSignificantValueDigits, boolean keyed, DocValueFormat format, + List pipelineAggregators, Map metaData) throws IOException { + super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, format, pipelineAggregators, + metaData); + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + DoubleHistogram state = getState(owningBucketOrdinal); + if (state == null) { + return buildEmptyAggregation(); + } else { + return new InternalHDRPercentileRanks(name, keys, state, keyed, format, pipelineAggregators(), metaData()); + } + } + + @Override + public InternalAggregation buildEmptyAggregation() { + DoubleHistogram state; + state = new DoubleHistogram(numberOfSignificantValueDigits); + state.setAutoResize(true); + return new InternalHDRPercentileRanks(name, keys, state, + keyed, format, pipelineAggregators(), metaData()); + } + + @Override + public double metric(String name, long bucketOrd) { + DoubleHistogram state = getState(bucketOrd); + if (state == null) { + return Double.NaN; + } else { + return InternalHDRPercentileRanks.percentileRank(state, Double.valueOf(name)); + } + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java new file mode 100644 index 0000000000000..283f07fcc71f4 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.HdrHistogram.DoubleHistogram; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentiles; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class HistoBackedHDRPercentilesAggregator extends AbstractHistoBackedHDRPercentilesAggregator { + + HistoBackedHDRPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, double[] percents, + int numberOfSignificantValueDigits, boolean keyed, DocValueFormat formatter, + List pipelineAggregators, Map metaData) throws IOException { + super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, formatter, + pipelineAggregators, metaData); + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + DoubleHistogram state = getState(owningBucketOrdinal); + if (state == null) { + return buildEmptyAggregation(); + } else { + return new InternalHDRPercentiles(name, keys, state, keyed, format, pipelineAggregators(), metaData()); + } + } + + @Override + public double metric(String name, long bucketOrd) { + DoubleHistogram state = getState(bucketOrd); + if (state == null) { + return Double.NaN; + } else { + return state.getValueAtPercentile(Double.parseDouble(name)); + } + } + + @Override + public InternalAggregation buildEmptyAggregation() { + DoubleHistogram state; + state = new DoubleHistogram(numberOfSignificantValueDigits); + state.setAutoResize(true); + return new InternalHDRPercentiles(name, keys, state, + keyed, + format, pipelineAggregators(), metaData()); + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java new file mode 100644 index 0000000000000..7bd1f9d35e995 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +class HistoBackedTDigestPercentileRanksAggregator extends AbstractHistoBackedTDigestPercentilesAggregator { + + HistoBackedTDigestPercentileRanksAggregator(String name, + ValuesSource valuesSource, + SearchContext context, + Aggregator parent, + double[] percents, + double compression, + boolean keyed, + DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException { + super(name, valuesSource, context, parent, percents, compression, keyed, formatter, pipelineAggregators, metaData); + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + TDigestState state = getState(owningBucketOrdinal); + if (state == null) { + return buildEmptyAggregation(); + } else { + return new InternalTDigestPercentileRanks(name, keys, state, keyed, formatter, pipelineAggregators(), metaData()); + } + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalTDigestPercentileRanks(name, keys, new TDigestState(compression), keyed, + formatter, pipelineAggregators(), metaData()); + } + + @Override + public double metric(String name, long bucketOrd) { + TDigestState state = getState(bucketOrd); + if (state == null) { + return Double.NaN; + } else { + return InternalTDigestPercentileRanks.percentileRank(state, Double.valueOf(name)); + } + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java new file mode 100644 index 0000000000000..6b7f33178ebf1 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.metrics; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +class HistoBackedTDigestPercentilesAggregator extends AbstractHistoBackedTDigestPercentilesAggregator { + + HistoBackedTDigestPercentilesAggregator(String name, + ValuesSource valuesSource, + SearchContext context, + Aggregator parent, + double[] percents, + double compression, + boolean keyed, + DocValueFormat formatter, + List pipelineAggregators, + Map metaData) throws IOException { + super(name, valuesSource, context, parent, percents, compression, keyed, formatter, pipelineAggregators, metaData); + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + TDigestState state = getState(owningBucketOrdinal); + if (state == null) { + return buildEmptyAggregation(); + } else { + return new InternalTDigestPercentiles(name, keys, state, keyed, formatter, pipelineAggregators(), metaData()); + } + } + + @Override + public double metric(String name, long bucketOrd) { + TDigestState state = getState(bucketOrd); + if (state == null) { + return Double.NaN; + } else { + return state.quantile(Double.parseDouble(name) / 100); + } + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalTDigestPercentiles(name, keys, new TDigestState(compression), keyed, + formatter, pipelineAggregators(), metaData()); + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java new file mode 100644 index 0000000000000..7c4bbfd8da13d --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.analytics.aggregations.support; + +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexHistogramFieldData; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.FieldContext; +import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + +import java.util.Locale; +import java.util.function.LongSupplier; + +public enum AnalyticsValuesSourceType implements ValuesSourceType { + HISTOGRAM() { + @Override + public ValuesSource getEmpty() { + // TODO: Is this the correct exception type here? + throw new IllegalArgumentException("Can't deal with unmapped HistogramValuesSource type " + this.value()); + } + + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + throw new AggregationExecutionException("value source of type [" + this.value() + "] is not supported by scripts"); + } + + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + final IndexFieldData indexFieldData = fieldContext.indexFieldData(); + + if (!(indexFieldData instanceof IndexHistogramFieldData)) { + throw new IllegalArgumentException("Expected histogram type on field [" + fieldContext.field() + + "], but got [" + fieldContext.fieldType().typeName() + "]"); + } + return new HistogramValuesSource.Histogram.Fielddata((IndexHistogramFieldData) indexFieldData); + } + + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + throw new IllegalArgumentException("Can't apply missing values on a " + valuesSource.getClass()); + } + }; + + + @Override + public boolean isCastableTo(ValuesSourceType valuesSourceType) { + // TODO right now only have to care about Histo, but in future we may + // have types that are castable to CoreValuesSourceTypes + if (valuesSourceType instanceof CoreValuesSourceType) { + return false; + } + AnalyticsValuesSourceType other = (AnalyticsValuesSourceType) valuesSourceType; + return other.value().equals(AnalyticsValuesSourceType.HISTOGRAM.value()); + } + + public static ValuesSourceType fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public String value() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java new file mode 100644 index 0000000000000..54a9582b4f34b --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.support; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.DocValueBits; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.index.fielddata.IndexHistogramFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; + +import java.io.IOException; + +public class HistogramValuesSource { + public abstract static class Histogram extends org.elasticsearch.search.aggregations.support.ValuesSource { + + public abstract HistogramValues getHistogramValues(LeafReaderContext context) throws IOException; + + public static class Fielddata extends Histogram { + + protected final IndexHistogramFieldData indexFieldData; + + public Fielddata(IndexHistogramFieldData indexFieldData) { + this.indexFieldData = indexFieldData; + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) { + return indexFieldData.load(context).getBytesValues(); + } + + @Override + public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { + HistogramValues values = getHistogramValues(context); + return new DocValueBits() { + @Override + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); + } + }; + } + + public HistogramValues getHistogramValues(LeafReaderContext context) throws IOException { + return indexFieldData.load(context).getHistogramValues(); + } + } + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregator.java index dec42ddbd6159..ec07e1642830e 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregator.java @@ -24,6 +24,7 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.analytics.aggregations.support.HistogramValuesSource; import java.io.IOException; import java.util.List; @@ -60,8 +61,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - if (valuesSource instanceof ValuesSource.Histogram) { - final HistogramValues values = ((ValuesSource.Histogram)valuesSource).getHistogramValues(ctx); + if (valuesSource instanceof HistogramValuesSource.Histogram) { + final HistogramValues values = ((HistogramValuesSource.Histogram)valuesSource).getHistogramValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java index a805335b1546b..975cb34465be1 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import java.io.IOException; import java.util.List; @@ -30,7 +31,7 @@ public class BoxplotAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry valuesSourceRegistry) { valuesSourceRegistry.register(BoxplotAggregationBuilder.NAME, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.HISTOGRAM), + List.of(CoreValuesSourceType.NUMERIC, AnalyticsValuesSourceType.HISTOGRAM), (BoxplotAggregatorSupplier) BoxplotAggregator::new); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 81b53a02b4eb6..fbdc48544b3da 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -34,11 +34,11 @@ import org.elasticsearch.index.fielddata.HistogramValue; import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -51,12 +51,10 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import java.io.IOException; import java.util.Iterator; @@ -282,8 +280,7 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu @Override public ValuesSourceType getValuesSourceType() { - // TODO: Histogram ValuesSourceType should move into this plugin. - return CoreValuesSourceType.HISTOGRAM; + return AnalyticsValuesSourceType.HISTOGRAM; } @Override diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java index aeba31c70a7bf..8dbd9c5fb96ee 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.junit.BeforeClass; import java.io.IOException; @@ -70,8 +71,7 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.NUMERIC, - CoreValuesSourceType.HISTOGRAM); + return List.of(CoreValuesSourceType.NUMERIC, AnalyticsValuesSourceType.HISTOGRAM); } @Override diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentileRanksAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentileRanksAggregatorTests.java index 843ff9447fde5..fab7bd91e0ea3 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentileRanksAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentileRanksAggregatorTests.java @@ -16,20 +16,49 @@ import org.apache.lucene.store.Directory; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.elasticsearch.search.aggregations.metrics.Percentile; import org.elasticsearch.search.aggregations.metrics.PercentileRanks; import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.hamcrest.Matchers; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Iterator; +import java.util.List; public class HDRPreAggregatedPercentileRanksAggregatorTests extends AggregatorTestCase { + @BeforeClass + public static void registerBuilder() { + AnalyticsPercentilesAggregatorFactory.registerPercentileRanksAggregator(valuesSourceRegistry); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentileRanksAggregationBuilder("hdr_percentiles", new double[]{1.0}) + .field(fieldName) + .percentilesConfig(new PercentilesConfig.Hdr()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + // Note: this is the same list as Core, plus Analytics + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + AnalyticsValuesSourceType.HISTOGRAM); + } + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { DoubleHistogram histogram = new DoubleHistogram(3);//default for (double value : values) { @@ -62,7 +91,7 @@ public void testSimple() throws IOException { fieldType.setName("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentilesAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentilesAggregatorTests.java index 5d4e5c05b2e9d..d71797039d78d 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentilesAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HDRPreAggregatedPercentilesAggregatorTests.java @@ -19,22 +19,51 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentiles; import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; public class HDRPreAggregatedPercentilesAggregatorTests extends AggregatorTestCase { - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { + @BeforeClass + public static void registerBuilder() { + AnalyticsPercentilesAggregatorFactory.registerPercentilesAggregator(valuesSourceRegistry); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentilesAggregationBuilder("hdr_percentiles") + .field(fieldName) + .percentilesConfig(new PercentilesConfig.Hdr()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + // Note: this is the same list as Core, plus Analytics + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + AnalyticsValuesSourceType.HISTOGRAM); + } + + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { DoubleHistogram histogram = new DoubleHistogram(3);//default for (double value : values) { histogram.recordValue(value); @@ -54,7 +83,7 @@ private BinaryDocValuesField getDocValue(String fieldName, double[] values) thro } return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } + } public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentileRanksAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentileRanksAggregatorTests.java index 879173a5bc26e..ad80192a3c67b 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentileRanksAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentileRanksAggregatorTests.java @@ -16,23 +16,52 @@ import org.apache.lucene.store.Directory; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.Percentile; import org.elasticsearch.search.aggregations.metrics.PercentileRanks; import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; import org.hamcrest.Matchers; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Collection; import java.util.Iterator; +import java.util.List; public class TDigestPreAggregatedPercentileRanksAggregatorTests extends AggregatorTestCase { + @BeforeClass + public static void registerBuilder() { + AnalyticsPercentilesAggregatorFactory.registerPercentileRanksAggregator(valuesSourceRegistry); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentileRanksAggregationBuilder("tdigest_percentiles", new double[]{1.0}) + .field(fieldName) + .percentilesConfig(new PercentilesConfig.TDigest()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + // Note: this is the same list as Core, plus Analytics + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + AnalyticsValuesSourceType.HISTOGRAM); + } + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { TDigest histogram = new TDigestState(100.0); //default for (double value : values) { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentilesAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentilesAggregatorTests.java index e1340619256cf..6b084535addb8 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentilesAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestPreAggregatedPercentilesAggregatorTests.java @@ -19,24 +19,53 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles; import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesConfig; import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsPercentilesAggregatorFactory; +import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTestCase { - private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { + @BeforeClass + public static void registerBuilder() { + AnalyticsPercentilesAggregatorFactory.registerPercentilesAggregator(valuesSourceRegistry); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentilesAggregationBuilder("tdigest_percentiles") + .field(fieldName) + .percentilesConfig(new PercentilesConfig.TDigest()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + // Note: this is the same list as Core, plus Analytics + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + AnalyticsValuesSourceType.HISTOGRAM); + } + + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { TDigest histogram = new TDigestState(100.0); //default for (double value : values) { histogram.add(value); @@ -51,7 +80,7 @@ private BinaryDocValuesField getDocValue(String fieldName, double[] values) thro streamOutput.writeDouble(centroid.mean()); } return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); - } + } public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> {