From d63ae90c777ffcd512a1bd781ffc9e10e4416e80 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 29 Dec 2021 15:36:48 +0100 Subject: [PATCH 01/36] feature: ipv4 and ipv6 subnet aggregator --- .../elasticsearch/search/DocValueFormat.java | 4 +- .../bucket/range/InternalIpPrefix.java | 300 +++++++++++++++ .../aggregations/bucket/range/IpPrefix.java | 26 ++ .../range/IpPrefixAggregationBuilder.java | 354 ++++++++++++++++++ .../range/IpPrefixAggregationSupplier.java | 36 ++ .../bucket/range/IpPrefixAggregator.java | 260 +++++++++++++ .../range/IpPrefixAggregatorFactory.java | 93 +++++ 7 files changed, 1071 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 58cf9c9352a6d..8b45f2f230c0b 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -443,14 +443,14 @@ public double parseDouble(String value, boolean roundUp, LongSupplier now) { } }; - DocValueFormat IP = IpDocValueFormat.INSTANCE; + IpDocValueFormat IP = IpDocValueFormat.INSTANCE; /** * Stateless, singleton formatter for IP address data */ class IpDocValueFormat implements DocValueFormat { - public static final DocValueFormat INSTANCE = new IpDocValueFormat(); + public static final IpDocValueFormat INSTANCE = new IpDocValueFormat(); private IpDocValueFormat() {} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java new file mode 100644 index 0000000000000..e134620f57242 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java @@ -0,0 +1,300 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.aggregations.bucket.range; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PriorityQueue; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationReduceContext; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; +import org.elasticsearch.search.aggregations.KeyComparable; +import org.elasticsearch.search.aggregations.bucket.IteratorAndCurrent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class InternalIpPrefix extends InternalMultiBucketAggregation { + + public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements IpPrefix.Bucket, KeyComparable { + + private final transient DocValueFormat format; + private final transient boolean keyed; + private final String key; + private final BytesRef value; + private final long docCount; + private final InternalAggregations aggregations; + + public Bucket( + DocValueFormat format, + boolean keyed, + String key, + BytesRef value, + long docCount, + InternalAggregations aggregations + ) { + this.format = format; + this.keyed = keyed; + this.key = key != null ? key : DocValueFormat.IP.format(value); + this.value = value; + this.docCount = docCount; + this.aggregations = aggregations; + } + + private static InternalIpPrefix.Bucket createFromStream(StreamInput in, DocValueFormat format, boolean keyed) throws IOException { + String key = in.readString(); + BytesRef value = in.readBytesRef(); + long docCount = in.readLong(); + InternalAggregations aggregations = InternalAggregations.readFrom(in); + + return new InternalIpPrefix.Bucket(format, keyed, key, value, docCount, aggregations); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (keyed) { + builder.startObject(key); + } else { + builder.startObject(); + builder.field(CommonFields.KEY.getPreferredName(), DocValueFormat.IP.format(this.value)); + } + builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); + aggregations.toXContentInternal(builder, params); + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(key); + out.writeBytesRef(value); + out.writeLong(docCount); + aggregations.writeTo(out); + } + + public DocValueFormat getFormat() { + return format; + } + + public boolean isKeyed() { + return keyed; + } + + public String getKey() { + return key; + } + + public BytesRef getValue() { + return value; + } + + @Override + public String getKeyAsString() { + return key; + } + + @Override + public long getDocCount() { + return docCount; + } + + @Override + public InternalAggregations getAggregations() { + return aggregations; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Bucket bucket = (Bucket) o; + return keyed == bucket.keyed && docCount == bucket.docCount && + Objects.equals(format, bucket.format) && Objects.equals(value, bucket.value) && + Objects.equals(aggregations, bucket.aggregations); + } + + @Override + public int hashCode() { + return Objects.hash(format, keyed, value, docCount, aggregations); + } + + @Override + public int compareKey(Bucket other) { + return this.value.compareTo(other.value); + } + } + + protected final DocValueFormat format; + protected final long minDocCount; + protected final boolean keyed; + private final List buckets; + + public InternalIpPrefix( + String name, + DocValueFormat format, + long minDocCount, + List buckets, + boolean keyed, + Map metadata + ) { + super(name, metadata); + this.minDocCount = minDocCount; + this.format = format; + this.keyed = keyed; + this.buckets = buckets; + } + + public InternalIpPrefix(StreamInput in) throws IOException { + super(in); + format = in.readNamedWriteable(DocValueFormat.class); + minDocCount = in.readVLong(); + keyed = in.readBoolean(); + buckets = in.readList(stream -> InternalIpPrefix.Bucket.createFromStream(stream, format, keyed)); + } + + @Override + public String getWriteableName() { + return IpPrefixAggregationBuilder.NAME; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(format); + out.writeVLong(minDocCount); + out.writeBoolean(keyed); + out.writeList(buckets); + } + + @Override + public InternalAggregation reduce(List aggregations, AggregationReduceContext reduceContext) { + final PriorityQueue> pq = new PriorityQueue<>(aggregations.size()) { + @Override + protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) { + return a.current().value.compareTo(b.current().value) < 0; + } + }; + for (InternalAggregation aggregation : aggregations) { + InternalIpPrefix ipPrefix = (InternalIpPrefix) aggregation; + if (ipPrefix.buckets.isEmpty() == false) { + pq.add(new IteratorAndCurrent<>(ipPrefix.buckets.iterator())); + } + } + + List reducedBuckets = new ArrayList<>(); + if (pq.size() > 0) { + // list of buckets coming from different shards that have the same value + List currentBuckets = new ArrayList<>(); + BytesRef value = pq.top().current().value; + + do { + final IteratorAndCurrent top = pq.top(); + if (!top.current().value.equals(value)) { + final Bucket reduced = reduceBucket(currentBuckets, reduceContext); + if (reduced.getDocCount() >= minDocCount) { + reducedBuckets.add(reduced); + } + currentBuckets.clear(); + value = top.current().value; + } + + currentBuckets.add(top.current()); + + if (top.hasNext()) { + top.next(); + assert top.current().value.compareTo(value) > 0 : + "shards must return data sorted by value [" + top.current().value + "] and [" + value + "]"; + pq.updateTop(); + } else { + pq.pop(); + } + } while (pq.size() > 0); + + if (currentBuckets.isEmpty() == false) { + final Bucket reduced = reduceBucket(currentBuckets, reduceContext); + if (reduced.getDocCount() >= minDocCount) { + reducedBuckets.add(reduced); + } + } + } + + return new InternalIpPrefix( + getName(), + format, + minDocCount, + reducedBuckets, + keyed, + metadata + ); + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + if (keyed) { + builder.startObject(CommonFields.BUCKETS.getPreferredName()); + } else { + builder.startArray(CommonFields.BUCKETS.getPreferredName()); + } + for (InternalIpPrefix.Bucket bucket : buckets) { + bucket.toXContent(builder, params); + } + if (keyed) { + builder.endObject(); + } else { + builder.endArray(); + } + return builder; + } + + @Override + public InternalIpPrefix create(List buckets) { + return new InternalIpPrefix(name, format, minDocCount, buckets, keyed, metadata); + } + + @Override + public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) { + return new Bucket(format, keyed, prototype.key, prototype.value, prototype.docCount, prototype.aggregations); + } + + @Override + protected Bucket reduceBucket(List buckets, AggregationReduceContext context) { + assert buckets.size() > 0; + List aggregations = new ArrayList<>(buckets.size()); + for (InternalIpPrefix.Bucket bucket : buckets) { + aggregations.add(bucket.getAggregations()); + } + InternalAggregations aggs = InternalAggregations.reduce(aggregations, context); + return createBucket(aggs, buckets.get(0)); + } + + @Override + public List getBuckets() { + return Collections.unmodifiableList(buckets); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + InternalIpPrefix that = (InternalIpPrefix) o; + return minDocCount == that.minDocCount && keyed == that.keyed && Objects.equals(format, that.format) && Objects.equals(buckets, that.buckets); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), format, minDocCount, keyed, buckets); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java new file mode 100644 index 0000000000000..ac89a787404ea --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.aggregations.bucket.range; + +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; + +import java.util.List; + +public interface IpPrefix extends MultiBucketsAggregation { + + interface Bucket extends MultiBucketsAggregation.Bucket { + + } + + /** + * Return the buckets of this range aggregation. + */ + @Override + List getBuckets(); +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java new file mode 100644 index 0000000000000..532819e3bf2d8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java @@ -0,0 +1,354 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.aggregations.bucket.range; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range.KEY_FIELD; + +public class IpPrefixAggregationBuilder extends ValuesSourceAggregationBuilder { + public static final String NAME = "ip_prefix"; + public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>( + NAME, + IpPrefixAggregationSupplier.class + ); + public static final ObjectParser PARSER = ObjectParser.fromBuilder( + NAME, + IpPrefixAggregationBuilder::new + ); + + private static final ParseField IS_IPV6 = new ParseField("is_ipv6"); + private static final ParseField PREFIX_LENGTH = new ParseField("prefix_len"); + + static { + ValuesSourceAggregationBuilder.declareFields(PARSER, false, false, false); + PARSER.declareBoolean(IpPrefixAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD); + PARSER.declareLong(IpPrefixAggregationBuilder::minDocCount, Histogram.MIN_DOC_COUNT_FIELD); + PARSER.declareObjectArray((agg, prefixes) -> { + for (IpPrefix prefix: prefixes) { + agg.addPrefix(prefix); + } + }, (p, c) -> IpPrefixAggregationBuilder.parseIpPrefix(p), RangeAggregator.RANGES_FIELD); + } + + public IpPrefixAggregationBuilder(StreamInput in) throws IOException { + super(in); + final int numPrefixes = in.readVInt(); + for (int i = 0; i < numPrefixes; ++i) { + addPrefix(new IpPrefix(in)); + } + keyed = in.readBoolean(); + isIpv6 = in.readBoolean(); + } + + private IpPrefixAggregationBuilder addPrefix(IpPrefix prefix) { + ipPrefixes.add(prefix); + return this; + } + + private static IpPrefix parseIpPrefix(XContentParser parser) throws IOException { + String key = null; + boolean isIpv6 = false; + String prefixLength = null; + + if (parser.currentToken() != XContentParser.Token.START_OBJECT) { + throw new ParsingException(parser.getTokenLocation(), "[ranges] must contain objects, but hit a " + parser.currentToken()); + } + while(parser.nextToken() != XContentParser.Token.END_OBJECT) { + if(parser.currentToken() == XContentParser.Token.FIELD_NAME) { + continue; + } + if (KEY_FIELD.match(parser.currentName(), parser.getDeprecationHandler())) { + key = parser.text(); + } else if (IS_IPV6.match(parser.currentName(), parser.getDeprecationHandler())) { + isIpv6 = Boolean.parseBoolean(parser.text()); + } else if (PREFIX_LENGTH.match(parser.currentName(), parser.getDeprecationHandler())) { + prefixLength = parser.text(); + } else { + throw new ParsingException(parser.getTokenLocation(), "Unexpected ip prefix parameter: [" + parser.currentName() + "]"); + } + } + if (prefixLength != null) { + if (key == null) { + key = prefixLength; + } + return new IpPrefix(key, isIpv6, Integer.parseInt(prefixLength)); + } + + throw new IllegalArgumentException("No [prefix_len] specified"); + } + + public static void registerAggregators(ValuesSourceRegistry.Builder builder) { + IpPrefixAggregatorFactory.registerAggregators(builder); + } + + public static class IpPrefix implements ToXContentObject { + private final String key; + private final boolean isIpv6; + private final Integer prefixLength; + + public IpPrefix(String key, boolean isIpv6, Integer prefixLength) { + this.key = key; + this.isIpv6 = isIpv6; + this.prefixLength = prefixLength; + } + + public IpPrefix(StreamInput in) throws IOException { + this.key = in.readOptionalString(); + this.isIpv6 = in.readBoolean(); + this.prefixLength = in.readVInt(); + } + + public String getKey() { + return key; + } + + public boolean isIpv6() { + return isIpv6; + } + + public Integer getPrefixLength() { + return prefixLength; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IpPrefix ipPrefix = (IpPrefix) o; + return isIpv6 == ipPrefix.isIpv6 && Objects.equals(key, ipPrefix.key) && Objects.equals(prefixLength, ipPrefix.prefixLength); + } + + @Override + public int hashCode() { + return Objects.hash(key, isIpv6, prefixLength); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (key != null) { + builder.field(KEY_FIELD.getPreferredName(), key); + } + if (prefixLength != null) { + builder.field(PREFIX_LENGTH.getPreferredName(), prefixLength); + } + builder.field(IS_IPV6.getPreferredName(), isIpv6); + builder.endObject(); + return builder; + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(key); + out.writeBoolean(isIpv6); + out.writeVInt(prefixLength); + } + } + + private boolean keyed = false; + private long minDocCount = 0; + private List ipPrefixes = new ArrayList<>(); + private boolean isIpv6 = false; + + public IpPrefixAggregationBuilder keyed(boolean keyed) { + this.keyed = keyed; + return this; + } + + public IpPrefixAggregationBuilder isIpv6(boolean isIpv6) { + this.isIpv6 = isIpv6; + return this; + } + + public IpPrefixAggregationBuilder minDocCount(long minDOcCount) { + this.minDocCount = minDOcCount; + return this; + } + + protected IpPrefixAggregationBuilder(String name) { + super(name); + } + + protected IpPrefixAggregationBuilder( + IpPrefixAggregationBuilder clone, + AggregatorFactories.Builder factoriesBuilder, + Map metadata + ) { + super(clone, factoriesBuilder, metadata); + this.ipPrefixes = new ArrayList<>(clone.ipPrefixes); + this.keyed = clone.keyed; + this.isIpv6 = clone.isIpv6; + } + + @Override + protected AggregationBuilder shallowCopy( + AggregatorFactories.Builder factoriesBuilder, + Map metadata + ) { + return new IpPrefixAggregationBuilder(this, factoriesBuilder, metadata); + } + + @Override + public BucketCardinality bucketCardinality() { + return BucketCardinality.MANY; + } + + @Override + public String getType() { + return NAME; + } + + @Override + protected void innerWriteTo(StreamOutput out) throws IOException { + out.writeVInt(ipPrefixes.size()); + for (IpPrefix ipPrefix: ipPrefixes) { + ipPrefix.writeTo(out); + } + out.writeBoolean(keyed); + out.writeBoolean(isIpv6); + } + + @Override + protected ValuesSourceRegistry.RegistryKey getRegistryKey() { + return REGISTRY_KEY; + } + + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.IP; + } + + @Override + protected ValuesSourceAggregatorFactory innerBuild( + AggregationContext context, + ValuesSourceConfig config, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder + ) throws IOException { + IpPrefixAggregationSupplier aggregationSupplier = context.getValuesSourceRegistry().getAggregator(REGISTRY_KEY, config); + + List prefixes = new ArrayList<>(); + if (this.ipPrefixes.size() == 0) { + throw new IllegalArgumentException("No [prefix] specified for the [" + this.getName() + "] aggregation"); + } + for (IpPrefixAggregationBuilder.IpPrefix ipPrefix : this.ipPrefixes) { + byte[] subnet = extractSubnet(ipPrefix.prefixLength, ipPrefix.isIpv6); + if (subnet == null) { + throw new IllegalArgumentException( + "Unable to compute subnet for prefix length [" + ipPrefix.prefixLength + "] and ip version [" + (ipPrefix.isIpv6 ? "ipv6" : "ipv4") + "]" + ); + } + prefixes.add( + new IpPrefixAggregator.IpPrefix( + ipPrefix.key, + ipPrefix.isIpv6, + subnet + ) + ); + } + + return new IpPrefixAggregatorFactory( + name, + config, + minDocCount, + keyed, + prefixes, + context, + parent, + subFactoriesBuilder, + metadata, + aggregationSupplier + ); + } + + private static byte[] extractSubnet(int prefixLength, boolean isIpv6) { + if(prefixLength < 0 || (!isIpv6 && prefixLength > 32) || (isIpv6 && prefixLength > 128)) { + return null; + } + + byte[] ipv4Address = { 0, 0, 0, 0 }; + byte[] ipv6Address = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + byte[] ipAddress = (isIpv6) ? ipv6Address : ipv4Address; + int bytesCount = prefixLength / 8; + int bitsCount = prefixLength % 8; + int i = 0; + for(; i < bytesCount; i++) { + ipAddress[i] = (byte) 0xFF; + } + if(bitsCount > 0) { + int rem = 0; + for(int j = 0; j < bitsCount; j++) { + rem |= 1 << (7 - j); + } + ipAddress[i] = (byte) rem; + } + + try { + return InetAddress.getByAddress(ipAddress).getAddress(); + } catch(UnknownHostException e) { + return null; + } + } + + private static byte[] toIpv6(byte[] ipv4Address) { + byte[] result = new byte[16]; + System.arraycopy(ipv4Address, 0, result, 16 - ipv4Address.length, ipv4Address.length); + return result; + } + + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(RangeAggregator.RANGES_FIELD.getPreferredName(), ipPrefixes); + builder.field(RangeAggregator.KEYED_FIELD.getPreferredName(), keyed); + builder.field(IS_IPV6.getPreferredName(), isIpv6); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + IpPrefixAggregationBuilder that = (IpPrefixAggregationBuilder) o; + return keyed == that.keyed && isIpv6 == that.isIpv6 && Objects.equals(ipPrefixes, that.ipPrefixes); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), keyed, ipPrefixes, isIpv6); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java new file mode 100644 index 0000000000000..169c0be120f32 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.aggregations.bucket.range; + +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.CardinalityUpperBound; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSource; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface IpPrefixAggregationSupplier { + Aggregator build( + String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + long minDocCount, + List prefixes, + boolean keyed, + AggregationContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java new file mode 100644 index 0000000000000..e95df13f8c6cb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java @@ -0,0 +1,260 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.aggregations.bucket.range; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionException; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.CardinalityUpperBound; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; +import org.elasticsearch.search.aggregations.bucket.terms.BytesKeyedBucketOrds; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSource; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +public final class IpPrefixAggregator extends BucketsAggregator { + + public static class IpPrefix { + final String key; + final boolean isIpv6; + final byte[] mask; + + public IpPrefix( + String key, + boolean isIpv6, + byte[] mask + ) { + this.key = key; + this.isIpv6 = isIpv6; + this.mask = mask; + } + + public String getKey() { + return key; + } + + public boolean isIpv6() { + return isIpv6; + } + + public byte[] getMask() { + return mask; + } + } + + private final Comparator IP_PREFIX_COMPARATOR = ((Comparator) (ipA, ipB) -> Arrays.compare(ipA.mask, ipB.mask)) + .thenComparing((ipA, ipB) -> Boolean.compare(ipA.isIpv6, ipB.isIpv6)) + .thenComparing(Comparator.nullsLast(Comparator.comparing(IpPrefix::getKey))); + + private final boolean keyed; + private final IpPrefix[] ipPrefixes; + + final ValuesSource.Bytes valuesSource; + final DocValueFormat format; + private final long minDocCount; + private final BytesKeyedBucketOrds bucketOrds; + + public IpPrefixAggregator( + String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + long minDocCount, + List ipPrefixes, + boolean keyed, + AggregationContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + super(name, factories, context, parent, cardinality, metadata); + this.valuesSource = (ValuesSource.Bytes) valuesSource; + this.format = format; + this.minDocCount = minDocCount; + this.keyed = keyed; + this.bucketOrds = BytesKeyedBucketOrds.build(bigArrays(), cardinality); + this.ipPrefixes = ipPrefixes.toArray(new IpPrefix[0]); + Arrays.sort(this.ipPrefixes, IP_PREFIX_COMPARATOR); + } + + @Override + protected LeafBucketCollector getLeafCollector( + LeafReaderContext ctx, + LeafBucketCollector sub + ) throws IOException { + return valuesSource == null ? + LeafBucketCollector.NO_OP_COLLECTOR : new IpPrefixLeafCollector(sub, valuesSource.bytesValues(ctx), ipPrefixes); + } + + private class IpPrefixLeafCollector extends LeafBucketCollectorBase { + private final IpPrefix[] ipPrefixes; + private final LeafBucketCollector sub; + private final SortedBinaryDocValues values; + + public IpPrefixLeafCollector( + LeafBucketCollector sub, + SortedBinaryDocValues values, + IpPrefix[] ipPrefixes + ) { + super(sub, values); + this.sub = sub; + this.values = values; + this.ipPrefixes = ipPrefixes; + for (int i = 1; i < ipPrefixes.length; ++i) { + if (IP_PREFIX_COMPARATOR.compare(ipPrefixes[i - 1], ipPrefixes[i]) > 0) { + throw new IllegalArgumentException("Ip prefixes must be sorted"); + } + } + } + + @Override + public void collect( + int doc, + long owningBucketOrd + ) throws IOException { + if (values.advanceExact(doc)) { + int valuesCount = values.docValueCount(); + + byte[] previousSubnet = null; + for (int i = 0; i < valuesCount; ++i) { + BytesRef ipAddress = values.nextValue(); + for (IpPrefix ipPrefix : ipPrefixes) { + byte[] subnet = maskIpAddress(ipAddress.bytes, ipPrefix.mask); + if (Arrays.equals(subnet, previousSubnet)) { + continue; + } + long bucketOrd = bucketOrds.add(owningBucketOrd, new BytesRef(subnet)); + if (bucketOrd < 0) { + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } + previousSubnet = subnet; + } + } + } + } + + private byte[] maskIpAddress(byte[] ipAddress, byte[] subnetMask) { + //NOTE: ip addresses are always encoded to 16 bytes by IpFieldMapper + if (ipAddress.length != 16) { + throw new IllegalArgumentException("Invalid length for ip address [" + ipAddress.length + "]"); + } + if (subnetMask.length == 4) { + return mask(Arrays.copyOfRange(ipAddress, 12, 16), subnetMask); + } + if (subnetMask.length == 16) { + return mask(ipAddress, subnetMask); + } + + throw new IllegalArgumentException("Invalid length for subnet mask [" + subnetMask.length + "]"); + } + + private byte[] mask(byte[] ipAddress, byte[] subnetMask) { + byte[] subnet = new byte[ipAddress.length]; + for (int i = 0; i < ipAddress.length; ++i) { + subnet[i] = (byte) (ipAddress[i] & subnetMask[i]); + } + + return subnet; + } + } + + + @Override + public InternalAggregation[] buildAggregations( + long[] owningBucketOrds + ) throws IOException { + long totalOrdsToCollect = 0; + final int[] bucketsInOrd = new int[owningBucketOrds.length]; + for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) { + final long bucketCount = bucketOrds.bucketsInOrd(owningBucketOrds[ordIdx]); + bucketsInOrd[ordIdx] = (int) bucketCount; + totalOrdsToCollect += bucketCount; + } + + long[] bucketOrdsToCollect = new long[(int) totalOrdsToCollect]; + int b = 0; + for (long owningBucketOrd : owningBucketOrds) { + BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); + while (ordsEnum.next()) { + bucketOrdsToCollect[b++] = ordsEnum.ord(); + } + } + InternalAggregations[] subAggregationResults = buildSubAggsForBuckets(bucketOrdsToCollect); + InternalAggregation[] results = new InternalAggregation[owningBucketOrds.length]; + b = 0; + for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) { + List buckets = new ArrayList<>(bucketsInOrd[ordIdx]); + BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds[ordIdx]); + while (ordsEnum.next()) { + long ordinal = ordsEnum.ord(); + if (bucketOrdsToCollect[b] != ordinal) { + throw new AggregationExecutionException( + "Iteration order of [" + + bucketOrds + + "] changed without mutating. [" + + ordinal + + "] should have been [" + + bucketOrdsToCollect[b] + + "]" + ); + } + BytesRef ipAddress = new BytesRef(); + ordsEnum.readValue(ipAddress); + long docCount = bucketDocCount(ordinal); + buckets.add( + new InternalIpPrefix.Bucket( + format, + keyed, + null, //ipAddress.toString(), //TODO: #57964 (use the key from the request) + BytesRef.deepCopyOf(ipAddress), + docCount, + subAggregationResults[b++] + ) + ); + + // NOTE: the aggregator is expected to return sorted results + CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator()); + } + results[ordIdx] = new InternalIpPrefix(name, format, minDocCount, buckets, keyed, metadata()); + } + return results; + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalIpPrefix( + name, + format, + minDocCount, + Collections.emptyList(), + keyed, + metadata() + ); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java new file mode 100644 index 0000000000000..24fad8763f343 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.aggregations.bucket.range; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.CardinalityUpperBound; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class IpPrefixAggregatorFactory extends ValuesSourceAggregatorFactory { + private final long minDocCount; + private final boolean keyed; + private final List prefixes; + private final IpPrefixAggregationSupplier aggregationSupplier + ; + + public IpPrefixAggregatorFactory( + String name, + ValuesSourceConfig config, + long minDocCount, + boolean keyed, + List prefixes, + AggregationContext context, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metadata, + IpPrefixAggregationSupplier aggregationSupplier + ) throws IOException { + super(name, config, context, parent, subFactoriesBuilder, metadata); + this.minDocCount = minDocCount; + this.keyed = keyed; + this.prefixes = prefixes; + this.aggregationSupplier = aggregationSupplier; + } + + public static void registerAggregators(ValuesSourceRegistry.Builder builder) { + builder.register( + IpPrefixAggregationBuilder.REGISTRY_KEY, + CoreValuesSourceType.IP, + IpPrefixAggregator::new, + true + ); + } + + @Override + protected Aggregator createUnmapped(Aggregator parent, Map metadata) throws IOException { + return new IpPrefixAggregator( + name, + factories, + null, + config.format(), + minDocCount, + prefixes, + keyed, + context, + parent, + CardinalityUpperBound.NONE, + metadata + ); + } + + @Override + protected Aggregator doCreateInternal(Aggregator parent, CardinalityUpperBound cardinality, Map metadata) throws IOException { + return aggregationSupplier.build( + name, + factories, + config.getValuesSource(), + config.format(), + minDocCount, + prefixes, + keyed, + context, + parent, + cardinality, + metadata + ); + } +} From bfd6875efec15a09f9747ed82120ebada8d145c3 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 29 Dec 2021 18:30:02 +0100 Subject: [PATCH 02/36] feature: new version without ranges --- .../test/search.aggregation/450_ip_prefix.yml | 359 ++++++ .../elasticsearch/search/SearchModule.java | 8 + .../bucket/range/InternalIpPrefix.java | 176 ++- .../range/IpPrefixAggregationBuilder.java | 286 ++--- .../range/IpPrefixAggregationSupplier.java | 5 +- .../bucket/range/IpPrefixAggregator.java | 160 +-- .../range/IpPrefixAggregatorFactory.java | 34 +- .../aggregations/bucket/IpPrefixTests.java | 41 + .../bucket/range/IpPrefixAggregatorTests.java | 1064 +++++++++++++++++ 9 files changed, 1796 insertions(+), 337 deletions(-) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml new file mode 100644 index 0000000000000..daba9ce5e4aaa --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml @@ -0,0 +1,359 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_replicas: 0 + mappings: + properties: + ipv4: + type: ip + ipv6: + type: ip + value: + type: long + + - do: + bulk: + index: test + refresh: true + body: + - { "index": { } } + - { "ipv4": "192.168.1.10", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f10", "value": 10 } + - { "index": { } } + - { "ipv4": "192.168.1.12", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f12", "value": 20 } + - { "index": { } } + - { "ipv4": "192.168.1.33", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f33", "value": 40 } + - { "index": { } } + - { "ipv4": "192.168.1.10", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f10", "value": 20 } + - { "index": { } } + - { "ipv4": "192.168.1.33", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f33", "value": 70 } + - { "index": { } } + - { "ipv4": "192.168.2.41", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f41", "value": 20 } + - { "index": { } } + - { "ipv4": "192.168.2.10", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f10", "value": 30 } + - { "index": { } } + - { "ipv4": "192.168.2.23", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f23", "value": 50 } + - { "index": { } } + - { "ipv4": "192.168.2.41", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f41", "value": 60 } + - { "index": { } } + - { "ipv4": "192.168.2.10", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f10", "value": 10 } + +--- +"IPv4 prefix": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ipv4" + is_ipv6: false + prefix_len: 24 + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.ip_prefix.buckets: 2 } + - match: { aggregations.ip_prefix.buckets.0.key: "192.168.1.0" } + - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } + - is_false: aggregations.ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.ip_prefix.buckets.0.prefix_len: 24 } + - match: { aggregations.ip_prefix.buckets.0.netmask: "255.255.255.0" } + - match: { aggregations.ip_prefix.buckets.1.key: "192.168.2.0" } + - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } + - is_false: aggregations.ip_prefix.buckets.1.is_ipv6 + - match: { aggregations.ip_prefix.buckets.1.prefix_len: 24 } + - match: { aggregations.ip_prefix.buckets.1.netmask: "255.255.255.0" } + +--- +"IPv4 short prefix": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + first: + ip_prefix: + field: "ipv4" + is_ipv6: false + prefix_len: 13 + second: + ip_prefix: + field: "ipv4" + is_ipv6: false + prefix_len: 6 + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.first.buckets: 1 } + - match: { aggregations.first.buckets.0.key: "192.168.0.0" } + - match: { aggregations.first.buckets.0.doc_count: 10 } + - is_false: aggregations.first.buckets.0.is_ipv6 + - match: { aggregations.first.buckets.0.prefix_len: 13 } + - match: { aggregations.first.buckets.0.netmask: "255.248.0.0" } + - length: { aggregations.second.buckets: 1 } + - match: { aggregations.second.buckets.0.key: "192.0.0.0" } + - match: { aggregations.second.buckets.0.doc_count: 10 } + - is_false: aggregations.second.buckets.0.is_ipv6 + - match: { aggregations.second.buckets.0.prefix_len: 6 } + - match: { aggregations.second.buckets.0.netmask: "252.0.0.0" } + +--- +"IPv6 prefix": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ipv6" + is_ipv6: true + prefix_len: 64 + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.ip_prefix.buckets: 2 } + - match: { aggregations.ip_prefix.buckets.0.key: "2001:db8:a4f8:112a::" } + - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } + - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.0.netmask: null } + - match: { aggregations.ip_prefix.buckets.1.key: "2001:db8:a4f8:112c::" } + - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } + - is_true: aggregations.ip_prefix.buckets.1.is_ipv6 + - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.1.netmask: null } + +--- +"Invalid IPv4 prefix": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + catch: /\[prefix_len\] must be in range \[0, 32\] for aggregation \[ip_prefix\]/ + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ipv4" + is_ipv6: false + prefix_len: 44 + + +--- +"Invalid IPv6 prefix": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + catch: /\[prefix_len\] must be in range \[0, 128\] for aggregation \[ip_prefix\]/ + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ipv6" + is_ipv6: true + prefix_len: 170 + +--- +"IPv4 prefix sub aggregation": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + top_ip_prefix: + ip_prefix: + field: "ipv4" + is_ipv6: false + prefix_len: 16 + aggs: + sub_ip_prefix: + ip_prefix: + field: "ipv4" + is_ipv6: false + prefix_len: 24 + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.top_ip_prefix.buckets: 1 } + - match: { aggregations.top_ip_prefix.buckets.0.key: "192.168.0.0" } + - match: { aggregations.top_ip_prefix.buckets.0.doc_count: 10 } + - is_false: aggregations.top_ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.top_ip_prefix.buckets.0.prefix_len: 16 } + - match: { aggregations.top_ip_prefix.buckets.0.netmask: "255.255.0.0" } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.key: "192.168.1.0" } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.doc_count: 5 } + - is_false: aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.prefix_len: 24 } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.netmask: "255.255.255.0" } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.key: "192.168.2.0" } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.doc_count: 5 } + - is_false: aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.is_ipv6 + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.prefix_len: 24 } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.netmask: "255.255.255.0" } + +--- +"IPv6 prefix sub aggregation": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + top_ip_prefix: + ip_prefix: + field: "ipv6" + is_ipv6: true + prefix_len: 48 + aggs: + sub_ip_prefix: + ip_prefix: + field: "ipv6" + is_ipv6: true + prefix_len: 64 + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.top_ip_prefix.buckets: 1 } + - match: { aggregations.top_ip_prefix.buckets.0.key: "2001:db8:a4f8::" } + - match: { aggregations.top_ip_prefix.buckets.0.doc_count: 10 } + - is_true: aggregations.top_ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.top_ip_prefix.buckets.0.prefix_len: 48 } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.key: "2001:db8:a4f8:112a::" } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.doc_count: 5 } + - is_true: aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.netmask: null } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.key: "2001:db8:a4f8:112c::" } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.doc_count: 5 } + - is_true: aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.is_ipv6 + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.netmask: null } + +--- +"IPv6 prefix metric sub aggregation": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ipv6" + is_ipv6: true + prefix_len: 64 + aggs: + sum: + sum: + field: value + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.ip_prefix.buckets: 2 } + - match: { aggregations.ip_prefix.buckets.0.key: "2001:db8:a4f8:112a::" } + - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } + - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.0.netmask: null } + - match: { aggregations.ip_prefix.buckets.0.sum.value: 160 } + - match: { aggregations.ip_prefix.buckets.1.key: "2001:db8:a4f8:112c::" } + - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } + - is_true: aggregations.ip_prefix.buckets.1.is_ipv6 + - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.1.netmask: null } + - match: { aggregations.ip_prefix.buckets.1.sum.value: 170 } + +--- +"IPv4 prefix appended": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ipv4" + is_ipv6: false + prefix_len: 24 + append_prefix_len: true + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.ip_prefix.buckets: 2 } + - match: { aggregations.ip_prefix.buckets.0.key: "192.168.1.0/24" } + - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } + - is_false: aggregations.ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.ip_prefix.buckets.0.prefix_len: 24 } + - match: { aggregations.ip_prefix.buckets.0.netmask: "255.255.255.0" } + - match: { aggregations.ip_prefix.buckets.1.key: "192.168.2.0/24" } + - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } + - is_false: aggregations.ip_prefix.buckets.1.is_ipv6 + - match: { aggregations.ip_prefix.buckets.1.prefix_len: 24 } + - match: { aggregations.ip_prefix.buckets.1.netmask: "255.255.255.0" } + +--- +"IPv6 prefix appended": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ipv6" + is_ipv6: true + prefix_len: 64 + append_prefix_len: true + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.ip_prefix.buckets: 2 } + - match: { aggregations.ip_prefix.buckets.0.key: "2001:db8:a4f8:112a::/64" } + - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } + - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.0.netmask: null } + - match: { aggregations.ip_prefix.buckets.1.key: "2001:db8:a4f8:112c::/64" } + - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } + - is_true: aggregations.ip_prefix.buckets.1.is_ipv6 + - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.1.netmask: null } diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 7b7fa2e8b4abf..3a37632bde262 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -123,7 +123,9 @@ import org.elasticsearch.search.aggregations.bucket.range.InternalBinaryRange; import org.elasticsearch.search.aggregations.bucket.range.InternalDateRange; import org.elasticsearch.search.aggregations.bucket.range.InternalGeoDistance; +import org.elasticsearch.search.aggregations.bucket.range.InternalIpPrefix; import org.elasticsearch.search.aggregations.bucket.range.InternalRange; +import org.elasticsearch.search.aggregations.bucket.range.IpPrefixAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.IpRangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder; @@ -524,6 +526,12 @@ private ValuesSourceRegistry registerAggregations(List plugins) { .setAggregatorRegistrar(DateRangeAggregationBuilder::registerAggregators), builder ); + registerAggregation( + new AggregationSpec(IpPrefixAggregationBuilder.NAME, IpPrefixAggregationBuilder::new, IpPrefixAggregationBuilder.PARSER) + .addResultReader(InternalIpPrefix::new) + .setAggregatorRegistrar(IpPrefixAggregationBuilder::registerAggregators), + builder + ); registerAggregation( new AggregationSpec(IpRangeAggregationBuilder.NAME, IpRangeAggregationBuilder::new, IpRangeAggregationBuilder.PARSER) .addResultReader(InternalBinaryRange::new) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java index e134620f57242..041d8e22bb501 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collections; import java.util.List; import java.util.Map; @@ -30,57 +31,101 @@ public class InternalIpPrefix extends InternalMultiBucketAggregation { - public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements IpPrefix.Bucket, KeyComparable { + public static class Bucket extends InternalMultiBucketAggregation.InternalBucket + implements + IpPrefix.Bucket, + KeyComparable { private final transient DocValueFormat format; - private final transient boolean keyed; - private final String key; - private final BytesRef value; + private final BytesRef key; + private final boolean keyed; + private final boolean isIpv6; + private final int prefixLength; + private final boolean appendPrefixLength; private final long docCount; private final InternalAggregations aggregations; public Bucket( DocValueFormat format, + BytesRef key, boolean keyed, - String key, - BytesRef value, + boolean isIpv6, + int prefixLength, + boolean appendPrefixLength, long docCount, InternalAggregations aggregations ) { this.format = format; + this.key = key; this.keyed = keyed; - this.key = key != null ? key : DocValueFormat.IP.format(value); - this.value = value; + this.isIpv6 = isIpv6; + this.prefixLength = prefixLength; + this.appendPrefixLength = appendPrefixLength; this.docCount = docCount; this.aggregations = aggregations; } - private static InternalIpPrefix.Bucket createFromStream(StreamInput in, DocValueFormat format, boolean keyed) throws IOException { - String key = in.readString(); - BytesRef value = in.readBytesRef(); - long docCount = in.readLong(); - InternalAggregations aggregations = InternalAggregations.readFrom(in); - - return new InternalIpPrefix.Bucket(format, keyed, key, value, docCount, aggregations); + public Bucket(StreamInput in, DocValueFormat format, boolean keyed) throws IOException { + this.format = format; + this.keyed = keyed; + this.key = in.readBytesRef(); + this.isIpv6 = in.readBoolean(); + this.prefixLength = in.readVInt(); + this.appendPrefixLength = in.readBoolean(); + this.docCount = in.readLong(); + this.aggregations = InternalAggregations.readFrom(in); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + String key = DocValueFormat.IP.format(this.key); + if (appendPrefixLength) { + key = key + "/" + prefixLength; + } if (keyed) { builder.startObject(key); } else { builder.startObject(); - builder.field(CommonFields.KEY.getPreferredName(), DocValueFormat.IP.format(this.value)); + builder.field(CommonFields.KEY.getPreferredName(), key); + } + if (!isIpv6) { + builder.field("netmask", DocValueFormat.IP.format(netmask(prefixLength))); } builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); + builder.field(IpPrefixAggregationBuilder.IS_IPV6_FIELD.getPreferredName(), isIpv6); + builder.field(IpPrefixAggregationBuilder.PREFIX_LENGTH_FIELD.getPreferredName(), prefixLength); aggregations.toXContentInternal(builder, params); - return builder.endObject(); + builder.endObject(); + return builder; + } + + private static BytesRef netmask(int prefixLength) { + final BitSet bs = new BitSet(32); + bs.set(0, 32, false); + bs.set(32 - prefixLength, 32, true); + return new BytesRef(toBigEndian(toIpv4ByteArray(bs.toByteArray()))); + } + + private static byte[] toIpv4ByteArray(byte[] array) { + byte[] netmask = new byte[4]; + System.arraycopy(array, 0, netmask, 0, array.length); + return netmask; + } + + public static byte[] toBigEndian(byte[] array) { + byte[] result = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + result[array.length - i - 1] = array[i]; + } + return result; } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(key); - out.writeBytesRef(value); + out.writeBytesRef(key); + out.writeBoolean(isIpv6); + out.writeVInt(prefixLength); + out.writeBoolean(appendPrefixLength); out.writeLong(docCount); aggregations.writeTo(out); } @@ -89,21 +134,25 @@ public DocValueFormat getFormat() { return format; } - public boolean isKeyed() { - return keyed; + public BytesRef getKey() { + return key; } - public String getKey() { - return key; + @Override + public String getKeyAsString() { + return DocValueFormat.IP.format(key); } - public BytesRef getValue() { - return value; + public boolean isIpv6() { + return isIpv6; } - @Override - public String getKeyAsString() { - return key; + public int getPrefixLength() { + return prefixLength; + } + + public boolean appendPrefixLength() { + return appendPrefixLength; } @Override @@ -121,48 +170,52 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Bucket bucket = (Bucket) o; - return keyed == bucket.keyed && docCount == bucket.docCount && - Objects.equals(format, bucket.format) && Objects.equals(value, bucket.value) && - Objects.equals(aggregations, bucket.aggregations); + return isIpv6 == bucket.isIpv6 + && prefixLength == bucket.prefixLength + && appendPrefixLength == bucket.appendPrefixLength + && docCount == bucket.docCount + && Objects.equals(format, bucket.format) + && Objects.equals(key, bucket.key) + && Objects.equals(aggregations, bucket.aggregations); } @Override public int hashCode() { - return Objects.hash(format, keyed, value, docCount, aggregations); + return Objects.hash(format, key, isIpv6, prefixLength, appendPrefixLength, docCount, aggregations); } @Override public int compareKey(Bucket other) { - return this.value.compareTo(other.value); + return this.key.compareTo(other.key); } } protected final DocValueFormat format; - protected final long minDocCount; protected final boolean keyed; + protected final long minDocCount; private final List buckets; public InternalIpPrefix( String name, DocValueFormat format, + boolean keyed, long minDocCount, List buckets, - boolean keyed, Map metadata ) { super(name, metadata); + this.keyed = keyed; this.minDocCount = minDocCount; this.format = format; - this.keyed = keyed; this.buckets = buckets; } public InternalIpPrefix(StreamInput in) throws IOException { super(in); format = in.readNamedWriteable(DocValueFormat.class); - minDocCount = in.readVLong(); keyed = in.readBoolean(); - buckets = in.readList(stream -> InternalIpPrefix.Bucket.createFromStream(stream, format, keyed)); + minDocCount = in.readVLong(); + buckets = in.readList(stream -> new Bucket(stream, format, keyed)); } @Override @@ -173,17 +226,24 @@ public String getWriteableName() { @Override protected void doWriteTo(StreamOutput out) throws IOException { out.writeNamedWriteable(format); - out.writeVLong(minDocCount); out.writeBoolean(keyed); + out.writeVLong(minDocCount); out.writeList(buckets); } @Override public InternalAggregation reduce(List aggregations, AggregationReduceContext reduceContext) { + List reducedBuckets = reduceBuckets(aggregations, reduceContext); + reduceContext.consumeBucketsAndMaybeBreak(reducedBuckets.size()); + + return new InternalIpPrefix(getName(), format, keyed, minDocCount, reducedBuckets, metadata); + } + + private List reduceBuckets(List aggregations, AggregationReduceContext reduceContext) { final PriorityQueue> pq = new PriorityQueue<>(aggregations.size()) { @Override protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) { - return a.current().value.compareTo(b.current().value) < 0; + return a.current().key.compareTo(b.current().key) < 0; } }; for (InternalAggregation aggregation : aggregations) { @@ -197,25 +257,25 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent 0) { // list of buckets coming from different shards that have the same value List currentBuckets = new ArrayList<>(); - BytesRef value = pq.top().current().value; + BytesRef value = pq.top().current().key; do { final IteratorAndCurrent top = pq.top(); - if (!top.current().value.equals(value)) { + if (!top.current().key.equals(value)) { final Bucket reduced = reduceBucket(currentBuckets, reduceContext); if (reduced.getDocCount() >= minDocCount) { reducedBuckets.add(reduced); } currentBuckets.clear(); - value = top.current().value; + value = top.current().key; } currentBuckets.add(top.current()); if (top.hasNext()) { top.next(); - assert top.current().value.compareTo(value) > 0 : - "shards must return data sorted by value [" + top.current().value + "] and [" + value + "]"; + assert top.current().key.compareTo(value) > 0 + : "shards must return data sorted by value [" + top.current().key + "] and [" + value + "]"; pq.updateTop(); } else { pq.pop(); @@ -230,14 +290,7 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent buckets) { - return new InternalIpPrefix(name, format, minDocCount, buckets, keyed, metadata); + return new InternalIpPrefix(name, format, keyed, minDocCount, buckets, metadata); } @Override public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) { - return new Bucket(format, keyed, prototype.key, prototype.value, prototype.docCount, prototype.aggregations); + return new Bucket( + format, + prototype.key, + prototype.keyed, + prototype.isIpv6, + prototype.prefixLength, + prototype.appendPrefixLength, + prototype.docCount, + prototype.aggregations + ); } @Override @@ -290,11 +352,11 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; InternalIpPrefix that = (InternalIpPrefix) o; - return minDocCount == that.minDocCount && keyed == that.keyed && Objects.equals(format, that.format) && Objects.equals(buckets, that.buckets); + return minDocCount == that.minDocCount && Objects.equals(format, that.format) && Objects.equals(buckets, that.buckets); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), format, minDocCount, keyed, buckets); + return Objects.hash(super.hashCode(), format, minDocCount, buckets); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java index 532819e3bf2d8..fbda1346a4ffb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java @@ -8,14 +8,12 @@ package org.elasticsearch.search.aggregations.bucket.range; -import org.elasticsearch.common.ParsingException; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; @@ -25,20 +23,14 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Objects; -import static org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range.KEY_FIELD; - public class IpPrefixAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "ip_prefix"; public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>( @@ -50,141 +42,54 @@ public class IpPrefixAggregationBuilder extends ValuesSourceAggregationBuilder { - for (IpPrefix prefix: prefixes) { - agg.addPrefix(prefix); - } - }, (p, c) -> IpPrefixAggregationBuilder.parseIpPrefix(p), RangeAggregator.RANGES_FIELD); + PARSER.declareInt(IpPrefixAggregationBuilder::prefixLength, PREFIX_LENGTH_FIELD); + PARSER.declareBoolean(IpPrefixAggregationBuilder::isIpv6, IS_IPV6_FIELD); + PARSER.declareLong(IpPrefixAggregationBuilder::minDocCount, MIN_DOC_COUNT_FIELD); + PARSER.declareBoolean(IpPrefixAggregationBuilder::appendPrefixLength, APPEND_PREFIX_LENGTH_FIELD); + PARSER.declareBoolean(IpPrefixAggregationBuilder::keyed, KEYED_FIELD); } + private static final int IPV6_MAX_PREFIX_LENGTH = 128; + private static final int IPV4_MAX_PREFIX_LENGTH = 32; + private static final int MIN_PREFIX_LENGTH = 0; + public IpPrefixAggregationBuilder(StreamInput in) throws IOException { super(in); - final int numPrefixes = in.readVInt(); - for (int i = 0; i < numPrefixes; ++i) { - addPrefix(new IpPrefix(in)); - } - keyed = in.readBoolean(); - isIpv6 = in.readBoolean(); - } - - private IpPrefixAggregationBuilder addPrefix(IpPrefix prefix) { - ipPrefixes.add(prefix); - return this; - } - - private static IpPrefix parseIpPrefix(XContentParser parser) throws IOException { - String key = null; - boolean isIpv6 = false; - String prefixLength = null; - - if (parser.currentToken() != XContentParser.Token.START_OBJECT) { - throw new ParsingException(parser.getTokenLocation(), "[ranges] must contain objects, but hit a " + parser.currentToken()); - } - while(parser.nextToken() != XContentParser.Token.END_OBJECT) { - if(parser.currentToken() == XContentParser.Token.FIELD_NAME) { - continue; - } - if (KEY_FIELD.match(parser.currentName(), parser.getDeprecationHandler())) { - key = parser.text(); - } else if (IS_IPV6.match(parser.currentName(), parser.getDeprecationHandler())) { - isIpv6 = Boolean.parseBoolean(parser.text()); - } else if (PREFIX_LENGTH.match(parser.currentName(), parser.getDeprecationHandler())) { - prefixLength = parser.text(); - } else { - throw new ParsingException(parser.getTokenLocation(), "Unexpected ip prefix parameter: [" + parser.currentName() + "]"); - } - } - if (prefixLength != null) { - if (key == null) { - key = prefixLength; - } - return new IpPrefix(key, isIpv6, Integer.parseInt(prefixLength)); - } - - throw new IllegalArgumentException("No [prefix_len] specified"); + this.prefixLength = in.readVInt(); + this.isIpv6 = in.readBoolean(); + this.minDocCount = in.readVLong(); + this.appendPrefixLength = in.readBoolean(); + this.keyed = in.readBoolean(); } public static void registerAggregators(ValuesSourceRegistry.Builder builder) { IpPrefixAggregatorFactory.registerAggregators(builder); } - public static class IpPrefix implements ToXContentObject { - private final String key; - private final boolean isIpv6; - private final Integer prefixLength; - - public IpPrefix(String key, boolean isIpv6, Integer prefixLength) { - this.key = key; - this.isIpv6 = isIpv6; - this.prefixLength = prefixLength; - } - - public IpPrefix(StreamInput in) throws IOException { - this.key = in.readOptionalString(); - this.isIpv6 = in.readBoolean(); - this.prefixLength = in.readVInt(); - } - - public String getKey() { - return key; - } - - public boolean isIpv6() { - return isIpv6; - } - - public Integer getPrefixLength() { - return prefixLength; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - IpPrefix ipPrefix = (IpPrefix) o; - return isIpv6 == ipPrefix.isIpv6 && Objects.equals(key, ipPrefix.key) && Objects.equals(prefixLength, ipPrefix.prefixLength); - } - - @Override - public int hashCode() { - return Objects.hash(key, isIpv6, prefixLength); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - if (key != null) { - builder.field(KEY_FIELD.getPreferredName(), key); - } - if (prefixLength != null) { - builder.field(PREFIX_LENGTH.getPreferredName(), prefixLength); - } - builder.field(IS_IPV6.getPreferredName(), isIpv6); - builder.endObject(); - return builder; - } - - public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalString(key); - out.writeBoolean(isIpv6); - out.writeVInt(prefixLength); - } - } - - private boolean keyed = false; private long minDocCount = 0; - private List ipPrefixes = new ArrayList<>(); + private int prefixLength = -1; private boolean isIpv6 = false; + private boolean appendPrefixLength = false; + private boolean keyed = false; - public IpPrefixAggregationBuilder keyed(boolean keyed) { - this.keyed = keyed; + public IpPrefixAggregationBuilder minDocCount(long minDocCount) { + this.minDocCount = minDocCount; + return this; + } + + public IpPrefixAggregationBuilder prefixLength(int prefixLength) { + if (prefixLength < MIN_PREFIX_LENGTH) { + throw new IllegalArgumentException("[prefix_len] must not be less than " + MIN_PREFIX_LENGTH + ": [" + name + "]"); + } + this.prefixLength = prefixLength; return this; } @@ -193,12 +98,17 @@ public IpPrefixAggregationBuilder isIpv6(boolean isIpv6) { return this; } - public IpPrefixAggregationBuilder minDocCount(long minDOcCount) { - this.minDocCount = minDOcCount; + public IpPrefixAggregationBuilder appendPrefixLength(boolean appendPrefixLength) { + this.appendPrefixLength = appendPrefixLength; return this; } - protected IpPrefixAggregationBuilder(String name) { + public IpPrefixAggregationBuilder keyed(boolean keyed) { + this.keyed = keyed; + return this; + } + + public IpPrefixAggregationBuilder(String name) { super(name); } @@ -208,16 +118,15 @@ protected IpPrefixAggregationBuilder( Map metadata ) { super(clone, factoriesBuilder, metadata); - this.ipPrefixes = new ArrayList<>(clone.ipPrefixes); - this.keyed = clone.keyed; + this.minDocCount = clone.minDocCount; this.isIpv6 = clone.isIpv6; + this.prefixLength = clone.prefixLength; + this.appendPrefixLength = clone.appendPrefixLength; + this.keyed = clone.keyed; } @Override - protected AggregationBuilder shallowCopy( - AggregatorFactories.Builder factoriesBuilder, - Map metadata - ) { + protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metadata) { return new IpPrefixAggregationBuilder(this, factoriesBuilder, metadata); } @@ -233,12 +142,11 @@ public String getType() { @Override protected void innerWriteTo(StreamOutput out) throws IOException { - out.writeVInt(ipPrefixes.size()); - for (IpPrefix ipPrefix: ipPrefixes) { - ipPrefix.writeTo(out); - } - out.writeBoolean(keyed); + out.writeVInt(prefixLength); out.writeBoolean(isIpv6); + out.writeVLong(minDocCount); + out.writeBoolean(appendPrefixLength); + out.writeBoolean(keyed); } @Override @@ -260,32 +168,61 @@ protected ValuesSourceAggregatorFactory innerBuild( ) throws IOException { IpPrefixAggregationSupplier aggregationSupplier = context.getValuesSourceRegistry().getAggregator(REGISTRY_KEY, config); - List prefixes = new ArrayList<>(); - if (this.ipPrefixes.size() == 0) { - throw new IllegalArgumentException("No [prefix] specified for the [" + this.getName() + "] aggregation"); + if (isIpv6 && prefixLength > IPV6_MAX_PREFIX_LENGTH) { + throw new IllegalArgumentException( + "[" + + PREFIX_LENGTH_FIELD.getPreferredName() + + "] must be in range [" + + MIN_PREFIX_LENGTH + + ", " + + IPV6_MAX_PREFIX_LENGTH + + "] for aggregation [" + + this.getName() + + "]" + ); } - for (IpPrefixAggregationBuilder.IpPrefix ipPrefix : this.ipPrefixes) { - byte[] subnet = extractSubnet(ipPrefix.prefixLength, ipPrefix.isIpv6); - if (subnet == null) { - throw new IllegalArgumentException( - "Unable to compute subnet for prefix length [" + ipPrefix.prefixLength + "] and ip version [" + (ipPrefix.isIpv6 ? "ipv6" : "ipv4") + "]" - ); - } - prefixes.add( - new IpPrefixAggregator.IpPrefix( - ipPrefix.key, - ipPrefix.isIpv6, - subnet - ) + + if (!isIpv6 && prefixLength > IPV4_MAX_PREFIX_LENGTH) { + throw new IllegalArgumentException( + "[" + + PREFIX_LENGTH_FIELD.getPreferredName() + + "] must be in range [" + + MIN_PREFIX_LENGTH + + ", " + + IPV4_MAX_PREFIX_LENGTH + + "] for aggregation [" + + this.getName() + + "]" ); } + byte[] subnet = extractSubnet(prefixLength, isIpv6); + if (subnet == null) { + throw new IllegalArgumentException( + "[" + + PREFIX_LENGTH_FIELD.getPreferredName() + + "] must be in range [" + + MIN_PREFIX_LENGTH + + ", " + + IPV4_MAX_PREFIX_LENGTH + + "] for aggregation [" + + this.getName() + + "]" + ); + } + IpPrefixAggregator.IpPrefix ipPrefix = new IpPrefixAggregator.IpPrefix( + isIpv6, + prefixLength, + appendPrefixLength, + new BytesRef(subnet) + ); + return new IpPrefixAggregatorFactory( name, config, - minDocCount, keyed, - prefixes, + minDocCount, + ipPrefix, context, parent, subFactoriesBuilder, @@ -295,7 +232,7 @@ protected ValuesSourceAggregatorFactory innerBuild( } private static byte[] extractSubnet(int prefixLength, boolean isIpv6) { - if(prefixLength < 0 || (!isIpv6 && prefixLength > 32) || (isIpv6 && prefixLength > 128)) { + if (prefixLength < 0 || (!isIpv6 && prefixLength > 32) || (isIpv6 && prefixLength > 128)) { return null; } @@ -305,12 +242,12 @@ private static byte[] extractSubnet(int prefixLength, boolean isIpv6) { int bytesCount = prefixLength / 8; int bitsCount = prefixLength % 8; int i = 0; - for(; i < bytesCount; i++) { + for (; i < bytesCount; i++) { ipAddress[i] = (byte) 0xFF; } - if(bitsCount > 0) { + if (bitsCount > 0) { int rem = 0; - for(int j = 0; j < bitsCount; j++) { + for (int j = 0; j < bitsCount; j++) { rem |= 1 << (7 - j); } ipAddress[i] = (byte) rem; @@ -318,23 +255,18 @@ private static byte[] extractSubnet(int prefixLength, boolean isIpv6) { try { return InetAddress.getByAddress(ipAddress).getAddress(); - } catch(UnknownHostException e) { + } catch (UnknownHostException e) { return null; } } - private static byte[] toIpv6(byte[] ipv4Address) { - byte[] result = new byte[16]; - System.arraycopy(ipv4Address, 0, result, 16 - ipv4Address.length, ipv4Address.length); - return result; - } - - @Override protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { - builder.field(RangeAggregator.RANGES_FIELD.getPreferredName(), ipPrefixes); - builder.field(RangeAggregator.KEYED_FIELD.getPreferredName(), keyed); - builder.field(IS_IPV6.getPreferredName(), isIpv6); + builder.field(PREFIX_LENGTH_FIELD.getPreferredName(), prefixLength); + builder.field(IS_IPV6_FIELD.getPreferredName(), isIpv6); + builder.field(APPEND_PREFIX_LENGTH_FIELD.getPreferredName(), appendPrefixLength); + builder.field(KEYED_FIELD.getPreferredName(), keyed); + builder.field(MIN_DOC_COUNT_FIELD.getPreferredName(), minDocCount); return builder; } @@ -344,11 +276,11 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; IpPrefixAggregationBuilder that = (IpPrefixAggregationBuilder) o; - return keyed == that.keyed && isIpv6 == that.isIpv6 && Objects.equals(ipPrefixes, that.ipPrefixes); + return minDocCount == that.minDocCount && prefixLength == that.prefixLength && isIpv6 == that.isIpv6; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), keyed, ipPrefixes, isIpv6); + return Objects.hash(super.hashCode(), minDocCount, prefixLength, isIpv6); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java index 169c0be120f32..33907b3bc982a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java @@ -16,7 +16,6 @@ import org.elasticsearch.search.aggregations.support.ValuesSource; import java.io.IOException; -import java.util.List; import java.util.Map; public interface IpPrefixAggregationSupplier { @@ -25,9 +24,9 @@ Aggregator build( AggregatorFactories factories, ValuesSource valuesSource, DocValueFormat format, - long minDocCount, - List prefixes, boolean keyed, + long minDocCount, + IpPrefixAggregator.IpPrefix ipPrefix, AggregationContext context, Aggregator parent, CardinalityUpperBound cardinality, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java index e95df13f8c6cb..bbf06cd6018e6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.core.Releasables; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationExecutionException; @@ -31,136 +32,138 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Objects; public final class IpPrefixAggregator extends BucketsAggregator { public static class IpPrefix { - final String key; final boolean isIpv6; - final byte[] mask; + final int prefixLength; + final boolean appendPrefixLength; + final BytesRef mask; - public IpPrefix( - String key, - boolean isIpv6, - byte[] mask - ) { - this.key = key; + public IpPrefix(boolean isIpv6, int prefixLength, boolean appendPrefixLength, BytesRef mask) { this.isIpv6 = isIpv6; + this.prefixLength = prefixLength; + this.appendPrefixLength = appendPrefixLength; this.mask = mask; } - public String getKey() { - return key; - } - public boolean isIpv6() { return isIpv6; } - public byte[] getMask() { + public int getPrefixLength() { + return prefixLength; + } + + public boolean appendPrefixLength() { + return appendPrefixLength; + } + + public BytesRef getMask() { return mask; } - } - private final Comparator IP_PREFIX_COMPARATOR = ((Comparator) (ipA, ipB) -> Arrays.compare(ipA.mask, ipB.mask)) - .thenComparing((ipA, ipB) -> Boolean.compare(ipA.isIpv6, ipB.isIpv6)) - .thenComparing(Comparator.nullsLast(Comparator.comparing(IpPrefix::getKey))); + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IpPrefix ipPrefix = (IpPrefix) o; + return isIpv6 == ipPrefix.isIpv6 + && prefixLength == ipPrefix.prefixLength + && appendPrefixLength == ipPrefix.appendPrefixLength + && Objects.equals(mask, ipPrefix.mask); + } - private final boolean keyed; - private final IpPrefix[] ipPrefixes; + @Override + public int hashCode() { + return Objects.hash(isIpv6, prefixLength, appendPrefixLength, mask); + } + } final ValuesSource.Bytes valuesSource; final DocValueFormat format; - private final long minDocCount; - private final BytesKeyedBucketOrds bucketOrds; + final long minDocCount; + final boolean keyed; + final BytesKeyedBucketOrds bucketOrds; + final IpPrefix ipPrefix; public IpPrefixAggregator( String name, AggregatorFactories factories, ValuesSource valuesSource, DocValueFormat format, - long minDocCount, - List ipPrefixes, boolean keyed, + long minDocCount, + IpPrefix ipPrefix, AggregationContext context, Aggregator parent, CardinalityUpperBound cardinality, Map metadata ) throws IOException { - super(name, factories, context, parent, cardinality, metadata); + super(name, factories, context, parent, CardinalityUpperBound.MANY, metadata); this.valuesSource = (ValuesSource.Bytes) valuesSource; this.format = format; - this.minDocCount = minDocCount; this.keyed = keyed; + this.minDocCount = minDocCount; this.bucketOrds = BytesKeyedBucketOrds.build(bigArrays(), cardinality); - this.ipPrefixes = ipPrefixes.toArray(new IpPrefix[0]); - Arrays.sort(this.ipPrefixes, IP_PREFIX_COMPARATOR); + this.ipPrefix = ipPrefix; } @Override - protected LeafBucketCollector getLeafCollector( - LeafReaderContext ctx, - LeafBucketCollector sub - ) throws IOException { - return valuesSource == null ? - LeafBucketCollector.NO_OP_COLLECTOR : new IpPrefixLeafCollector(sub, valuesSource.bytesValues(ctx), ipPrefixes); + protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { + return valuesSource == null + ? LeafBucketCollector.NO_OP_COLLECTOR + : new IpPrefixLeafCollector(sub, valuesSource.bytesValues(ctx), ipPrefix); } private class IpPrefixLeafCollector extends LeafBucketCollectorBase { - private final IpPrefix[] ipPrefixes; + private final IpPrefix ipPrefix; private final LeafBucketCollector sub; private final SortedBinaryDocValues values; - public IpPrefixLeafCollector( - LeafBucketCollector sub, - SortedBinaryDocValues values, - IpPrefix[] ipPrefixes - ) { + public IpPrefixLeafCollector(LeafBucketCollector sub, SortedBinaryDocValues values, IpPrefix ipPrefix) { super(sub, values); this.sub = sub; this.values = values; - this.ipPrefixes = ipPrefixes; - for (int i = 1; i < ipPrefixes.length; ++i) { - if (IP_PREFIX_COMPARATOR.compare(ipPrefixes[i - 1], ipPrefixes[i]) > 0) { - throw new IllegalArgumentException("Ip prefixes must be sorted"); - } - } + this.ipPrefix = ipPrefix; } @Override - public void collect( - int doc, - long owningBucketOrd - ) throws IOException { + public void collect(int doc, long owningBucketOrd) throws IOException { if (values.advanceExact(doc)) { int valuesCount = values.docValueCount(); byte[] previousSubnet = null; for (int i = 0; i < valuesCount; ++i) { - BytesRef ipAddress = values.nextValue(); - for (IpPrefix ipPrefix : ipPrefixes) { - byte[] subnet = maskIpAddress(ipAddress.bytes, ipPrefix.mask); - if (Arrays.equals(subnet, previousSubnet)) { - continue; - } - long bucketOrd = bucketOrds.add(owningBucketOrd, new BytesRef(subnet)); - if (bucketOrd < 0) { - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); - } - previousSubnet = subnet; + BytesRef value = values.nextValue(); + byte[] ipAddress = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); + byte[] mask = Arrays.copyOfRange( + ipPrefix.mask.bytes, + ipPrefix.mask.offset, + ipPrefix.mask.offset + ipPrefix.mask.length + ); + byte[] subnet = maskIpAddress(ipAddress, mask); + if (Arrays.equals(subnet, previousSubnet)) { + continue; + } + long bucketOrd = bucketOrds.add(owningBucketOrd, new BytesRef(subnet)); + if (bucketOrd < 0) { + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); } + previousSubnet = subnet; } } } private byte[] maskIpAddress(byte[] ipAddress, byte[] subnetMask) { - //NOTE: ip addresses are always encoded to 16 bytes by IpFieldMapper + // NOTE: ip addresses are always encoded to 16 bytes by IpFieldMapper if (ipAddress.length != 16) { throw new IllegalArgumentException("Invalid length for ip address [" + ipAddress.length + "]"); } @@ -184,11 +187,8 @@ private byte[] mask(byte[] ipAddress, byte[] subnetMask) { } } - @Override - public InternalAggregation[] buildAggregations( - long[] owningBucketOrds - ) throws IOException { + public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { long totalOrdsToCollect = 0; final int[] bucketsInOrd = new int[owningBucketOrds.length]; for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) { @@ -230,9 +230,11 @@ public InternalAggregation[] buildAggregations( buckets.add( new InternalIpPrefix.Bucket( format, - keyed, - null, //ipAddress.toString(), //TODO: #57964 (use the key from the request) BytesRef.deepCopyOf(ipAddress), + keyed, + ipPrefix.isIpv6, + ipPrefix.prefixLength, + ipPrefix.appendPrefixLength, docCount, subAggregationResults[b++] ) @@ -241,20 +243,18 @@ public InternalAggregation[] buildAggregations( // NOTE: the aggregator is expected to return sorted results CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator()); } - results[ordIdx] = new InternalIpPrefix(name, format, minDocCount, buckets, keyed, metadata()); + results[ordIdx] = new InternalIpPrefix(name, format, keyed, minDocCount, buckets, metadata()); } return results; } @Override public InternalAggregation buildEmptyAggregation() { - return new InternalIpPrefix( - name, - format, - minDocCount, - Collections.emptyList(), - keyed, - metadata() - ); + return new InternalIpPrefix(name, format, keyed, minDocCount, Collections.emptyList(), metadata()); + } + + @Override + public void doClose() { + Releasables.close(bucketOrds); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java index 24fad8763f343..ea41835fbdcd1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java @@ -19,22 +19,20 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import java.io.IOException; -import java.util.List; import java.util.Map; public class IpPrefixAggregatorFactory extends ValuesSourceAggregatorFactory { - private final long minDocCount; private final boolean keyed; - private final List prefixes; - private final IpPrefixAggregationSupplier aggregationSupplier - ; + private final long minDocCount; + private final IpPrefixAggregator.IpPrefix ipPrefix; + private final IpPrefixAggregationSupplier aggregationSupplier; public IpPrefixAggregatorFactory( String name, ValuesSourceConfig config, - long minDocCount, boolean keyed, - List prefixes, + long minDocCount, + IpPrefixAggregator.IpPrefix ipPrefix, AggregationContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, @@ -42,19 +40,14 @@ public IpPrefixAggregatorFactory( IpPrefixAggregationSupplier aggregationSupplier ) throws IOException { super(name, config, context, parent, subFactoriesBuilder, metadata); - this.minDocCount = minDocCount; this.keyed = keyed; - this.prefixes = prefixes; + this.minDocCount = minDocCount; + this.ipPrefix = ipPrefix; this.aggregationSupplier = aggregationSupplier; } public static void registerAggregators(ValuesSourceRegistry.Builder builder) { - builder.register( - IpPrefixAggregationBuilder.REGISTRY_KEY, - CoreValuesSourceType.IP, - IpPrefixAggregator::new, - true - ); + builder.register(IpPrefixAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.IP, IpPrefixAggregator::new, true); } @Override @@ -64,9 +57,9 @@ protected Aggregator createUnmapped(Aggregator parent, Map metad factories, null, config.format(), - minDocCount, - prefixes, keyed, + minDocCount, + ipPrefix, context, parent, CardinalityUpperBound.NONE, @@ -75,15 +68,16 @@ protected Aggregator createUnmapped(Aggregator parent, Map metad } @Override - protected Aggregator doCreateInternal(Aggregator parent, CardinalityUpperBound cardinality, Map metadata) throws IOException { + protected Aggregator doCreateInternal(Aggregator parent, CardinalityUpperBound cardinality, Map metadata) + throws IOException { return aggregationSupplier.build( name, factories, config.getValuesSource(), config.format(), - minDocCount, - prefixes, keyed, + minDocCount, + ipPrefix, context, parent, cardinality, diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java new file mode 100644 index 0000000000000..b5943cbd21093 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.aggregations.bucket; + +import org.elasticsearch.search.aggregations.BaseAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.range.IpPrefixAggregationBuilder; + +import static org.hamcrest.Matchers.startsWith; + +public class IpPrefixTests extends BaseAggregationTestCase { + @Override + protected IpPrefixAggregationBuilder createTestAggregatorBuilder() { + final String name = randomAlphaOfLengthBetween(3, 10); + final IpPrefixAggregationBuilder factory = new IpPrefixAggregationBuilder(name); + boolean isIpv6 = randomBoolean(); + int prefixLength = isIpv6 ? randomIntBetween(1, 128) : randomIntBetween(1, 32); + factory.field(IP_FIELD_NAME); + + factory.appendPrefixLength(randomBoolean()); + factory.isIpv6(isIpv6); + factory.prefixLength(prefixLength); + factory.keyed(randomBoolean()); + factory.minDocCount(randomIntBetween(1, 3)); + + return factory; + } + + public void testNegativePrefixLength() { + final IpPrefixAggregationBuilder factory = new IpPrefixAggregationBuilder(randomAlphaOfLengthBetween(3, 10)); + factory.isIpv6(randomBoolean()); + + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> factory.prefixLength(randomIntBetween(-1000, -1))); + assertThat(ex.getMessage(), startsWith("[prefix_len] must not be less than 0")); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java new file mode 100644 index 0000000000000..9cecf1b57c908 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java @@ -0,0 +1,1064 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.aggregations.bucket.range; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; +import org.elasticsearch.search.aggregations.metrics.InternalSum; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; + +import java.io.IOException; +import java.net.InetAddress; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static java.util.Collections.singleton; + +public class IpPrefixAggregatorTests extends AggregatorTestCase { + + private static final class TestIpDataHolder { + private final String ipAddressAsString; + private final InetAddress ipAddress; + private final String subnetAsString; + private final InetAddress subnet; + private final int prefixLength; + private final long time; + + public TestIpDataHolder(final String ipAddressAsString, final String subnetAsString, final int prefixLength, final long time) { + this.ipAddressAsString = ipAddressAsString; + this.ipAddress = InetAddresses.forString(ipAddressAsString); + this.subnetAsString = subnetAsString; + this.subnet = InetAddresses.forString(subnetAsString); + this.prefixLength = prefixLength; + this.time = time; + } + + public String getIpAddressAsString() { + return ipAddressAsString; + } + + public InetAddress getIpAddress() { + return ipAddress; + } + + public InetAddress getSubnet() { + return subnet; + } + + public String getSubnetAsString() { + return subnetAsString; + } + + public int getPrefixLength() { + return prefixLength; + } + + public long getTime() { + return time; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestIpDataHolder that = (TestIpDataHolder) o; + return prefixLength == that.prefixLength + && time == that.time + && Objects.equals(ipAddressAsString, that.ipAddressAsString) + && Objects.equals(ipAddress, that.ipAddress) + && Objects.equals(subnetAsString, that.subnetAsString) + && Objects.equals(subnet, that.subnet); + } + + @Override + public int hashCode() { + return Objects.hash(ipAddressAsString, ipAddress, subnetAsString, subnet, prefixLength, time); + } + + @Override + public String toString() { + return "TestIpDataHolder{ipAddressAsString='%s', ipAddress=%s, subnetAsString='%s', subnet=%s, prefixLength=%d, time=%d}" + .formatted(ipAddressAsString, ipAddress, subnetAsString, subnet, prefixLength, time); + } + } + + public void testEmptyDocument() throws IOException { + // GIVEN + final int prefixLength = 16; + final String field = "ipv4"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = Collections.emptyList(); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertFalse(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + + assertTrue(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testIpv4Addresses() throws IOException { + // GIVEN + final int prefixLength = 16; + final String field = "ipv4"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = List.of( + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.117", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.10.27", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.169.0.88", "192.169.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.19.0.44", "10.19.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.122.2.67", "10.122.0.0", prefixLength, defaultTime()) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertFalse(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testIpv6Addresses() throws IOException { + // GIVEN + final int prefixLength = 64; + final String field = "ipv6"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(true) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = List.of( + new TestIpDataHolder("2001:db8:a4f8:112a:6001:0:12:7f2a", "2001:db8:a4f8:112a::", prefixLength, defaultTime()), + new TestIpDataHolder("2001:db8:a4f8:112a:7044:1f01:0:44f2", "2001:db8:a4f8:112a::", prefixLength, defaultTime()), + new TestIpDataHolder("2001:db8:a4ff:112a::7002:7ff2", "2001:db8:a4ff:112a::", prefixLength, defaultTime()), + new TestIpDataHolder("3007:db81:4b11:234f:1212:0:1:3", "3007:db81:4b11:234f::", prefixLength, defaultTime()), + new TestIpDataHolder("3007:db81:4b11:234f:7770:12f6:0:30", "3007:db81:4b11:234f::", prefixLength, defaultTime()) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertTrue(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testZeroPrefixLength() throws IOException { + // GIVEN + final int prefixLength = 0; + final String field = "ipv4"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = List.of( + new TestIpDataHolder("192.168.1.12", "0.0.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.12", "0.0.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.117", "0.0.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.10.27", "0.0.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.169.0.88", "0.0.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.19.0.44", "0.0.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.122.2.67", "0.0.0.0", prefixLength, defaultTime()) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertFalse(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testIpv4MaxPrefixLength() throws IOException { + // GIVEN + final int prefixLength = 32; + final String field = "ipv4"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = List.of( + new TestIpDataHolder("192.168.1.12", "192.168.1.12", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.12", "192.168.1.12", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.117", "192.168.1.117", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.10.27", "192.168.10.27", prefixLength, defaultTime()), + new TestIpDataHolder("192.169.0.88", "192.169.0.88", prefixLength, defaultTime()), + new TestIpDataHolder("10.19.0.44", "10.19.0.44", prefixLength, defaultTime()), + new TestIpDataHolder("10.122.2.67", "10.122.2.67", prefixLength, defaultTime()) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertFalse(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testIpv6MaxPrefixLength() throws IOException { + // GIVEN + final int prefixLength = 128; + final String field = "ipv6"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(true) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = List.of( + new TestIpDataHolder("2001:db8:a4f8:112a:6001:0:12:7f2a", "2001:db8:a4f8:112a:6001:0:12:7f2a", prefixLength, defaultTime()), + new TestIpDataHolder("2001:db8:a4f8:112a:7044:1f01:0:44f2", "2001:db8:a4f8:112a:7044:1f01:0:44f2", prefixLength, defaultTime()), + new TestIpDataHolder("2001:db8:a4ff:112a:0:0:7002:7ff2", "2001:db8:a4ff:112a::7002:7ff2", prefixLength, defaultTime()), + new TestIpDataHolder("3007:db81:4b11:234f:1212:0:1:3", "3007:db81:4b11:234f:1212:0:1:3", prefixLength, defaultTime()), + new TestIpDataHolder("3007:db81:4b11:234f:7770:12f6:0:30", "3007:db81:4b11:234f:7770:12f6:0:30", prefixLength, defaultTime()) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertTrue(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testAggregateOnIpv4Field() throws IOException { + // GIVEN + final int prefixLength = 16; + final String ipv4FieldName = "ipv4"; + final String ipv6FieldName = "ipv6"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(ipv4FieldName) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType[] fieldTypes = { new IpFieldMapper.IpFieldType(ipv4FieldName), new IpFieldMapper.IpFieldType(ipv6FieldName) }; + final List ipAddresses = List.of( + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.117", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.10.27", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.169.0.88", "192.169.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.19.0.44", "10.19.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.122.2.67", "10.122.0.0", prefixLength, defaultTime()) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + List.of( + new SortedDocValuesField(ipv4FieldName, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress()))), + new SortedDocValuesField( + ipv6FieldName, + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("2001:db8:a4f8:112a:6001:0:12:7f2a"))) + ) + ) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertFalse(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldTypes); + } + + public void testAggregateOnIpv6Field() throws IOException { + // GIVEN + final int prefixLength = 64; + final String ipv4FieldName = "ipv4"; + final String ipv6FieldName = "ipv6"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(ipv6FieldName) + .isIpv6(true) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType[] fieldTypes = { new IpFieldMapper.IpFieldType(ipv4FieldName), new IpFieldMapper.IpFieldType(ipv6FieldName) }; + final List ipAddresses = List.of( + new TestIpDataHolder("2001:db8:a4f8:112a:6001:0:12:7f2a", "2001:db8:a4f8:112a::", prefixLength, defaultTime()), + new TestIpDataHolder("2001:db8:a4f8:112a:7044:1f01:0:44f2", "2001:db8:a4f8:112a::", prefixLength, defaultTime()), + new TestIpDataHolder("2001:db8:a4ff:112a::7002:7ff2", "2001:db8:a4ff:112a::", prefixLength, defaultTime()), + new TestIpDataHolder("3007:db81:4b11:234f:1212:0:1:3", "3007:db81:4b11:234f::", prefixLength, defaultTime()), + new TestIpDataHolder("3007:db81:4b11:234f:7770:12f6:0:30", "3007:db81:4b11:234f::", prefixLength, defaultTime()) + ); + final String ipv4Value = "192.168.10.20"; + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + List.of( + new SortedDocValuesField(ipv6FieldName, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress()))), + new SortedDocValuesField(ipv4FieldName, new BytesRef(InetAddressPoint.encode(InetAddresses.forString(ipv4Value)))) + ) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertTrue(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldTypes); + } + + public void testIpv4AggregationAsSubAggregation() throws IOException { + // GIVEN + final int prefixLength = 16; + final String ipv4FieldName = "ipv4"; + final String datetimeFieldName = "datetime"; + final String dateHistogramAggregationName = "date_histogram"; + final String ipPrefixAggregationName = "ip_prefix"; + final AggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder(dateHistogramAggregationName).calendarInterval( + DateHistogramInterval.DAY + ) + .field(datetimeFieldName) + .subAggregation( + new IpPrefixAggregationBuilder(ipPrefixAggregationName).field(ipv4FieldName) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength) + ); + final DateFieldMapper.DateFieldType dateFieldType = new DateFieldMapper.DateFieldType(datetimeFieldName); + final IpFieldMapper.IpFieldType ipFieldType = new IpFieldMapper.IpFieldType(ipv4FieldName); + final MappedFieldType[] fieldTypes = { ipFieldType, dateFieldType }; + + long day1 = dateFieldType.parse("2021-10-12"); + long day2 = dateFieldType.parse("2021-10-11"); + final List ipAddresses = List.of( + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, day1), + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, day2), + new TestIpDataHolder("192.168.1.117", "192.168.0.0", prefixLength, day1), + new TestIpDataHolder("192.168.10.27", "192.168.0.0", prefixLength, day2), + new TestIpDataHolder("192.169.0.88", "192.169.0.0", prefixLength, day1), + new TestIpDataHolder("10.19.0.44", "10.19.0.0", prefixLength, day2), + new TestIpDataHolder("10.122.2.67", "10.122.0.0", prefixLength, day1), + new TestIpDataHolder("10.19.13.32", "10.19.0.0", prefixLength, day2) + ); + + final Set expectedBucket1Subnets = ipAddresses.stream() + .filter(testIpDataHolder -> testIpDataHolder.getTime() == day1) + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set expectedBucket2Subnets = ipAddresses.stream() + .filter(testIpDataHolder -> testIpDataHolder.getTime() == day2) + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + + // WHEN + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (final TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + List.of( + new SortedDocValuesField(ipv4FieldName, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress()))), + new SortedNumericDocValuesField(datetimeFieldName, ipDataHolder.getTime()) + ) + ); + } + }, agg -> { + final InternalDateHistogram dateHistogram = (InternalDateHistogram) agg; + final List buckets = dateHistogram.getBuckets(); + assertEquals(2, buckets.size()); + + final InternalDateHistogram.Bucket day1Bucket = buckets.stream() + .filter(bucket -> bucket.getKey().equals(Instant.ofEpochMilli(day1).atZone(ZoneOffset.UTC))) + .findAny() + .orElse(null); + final InternalDateHistogram.Bucket day2Bucket = buckets.stream() + .filter(bucket -> bucket.getKey().equals(Instant.ofEpochMilli(day2).atZone(ZoneOffset.UTC))) + .findAny() + .orElse(null); + final InternalIpPrefix ipPrefix1 = Objects.requireNonNull(day1Bucket).getAggregations().get(ipPrefixAggregationName); + final InternalIpPrefix ipPrefix2 = Objects.requireNonNull(day2Bucket).getAggregations().get(ipPrefixAggregationName); + assertNotNull(ipPrefix1); + assertNotNull(ipPrefix2); + assertEquals(expectedBucket1Subnets.size(), ipPrefix1.getBuckets().size()); + assertEquals(expectedBucket2Subnets.size(), ipPrefix2.getBuckets().size()); + + final Set bucket1Subnets = ipPrefix1.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set bucket2Subnets = ipPrefix2.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + assertTrue(bucket1Subnets.containsAll(expectedBucket1Subnets)); + assertTrue(bucket2Subnets.containsAll(expectedBucket2Subnets)); + assertTrue(expectedBucket1Subnets.containsAll(bucket1Subnets)); + assertTrue(expectedBucket2Subnets.containsAll(bucket2Subnets)); + }, fieldTypes); + } + + public void testIpv6AggregationAsSubAggregation() throws IOException { + // GIVEN + final int prefixLength = 64; + final String ipv4FieldName = "ipv6"; + final String datetimeFieldName = "datetime"; + final String dateHistogramAggregationName = "date_histogram"; + final String ipPrefixAggregationName = "ip_prefix"; + final AggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder(dateHistogramAggregationName).calendarInterval( + DateHistogramInterval.DAY + ) + .field(datetimeFieldName) + .subAggregation( + new IpPrefixAggregationBuilder(ipPrefixAggregationName).field(ipv4FieldName) + .isIpv6(true) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength) + ); + final DateFieldMapper.DateFieldType dateFieldType = new DateFieldMapper.DateFieldType(datetimeFieldName); + final IpFieldMapper.IpFieldType ipFieldType = new IpFieldMapper.IpFieldType(ipv4FieldName); + final MappedFieldType[] fieldTypes = { ipFieldType, dateFieldType }; + + long day1 = dateFieldType.parse("2021-11-04"); + long day2 = dateFieldType.parse("2021-11-05"); + final List ipAddresses = List.of( + new TestIpDataHolder("2001:db8:a4f8:112a:6001:0:12:7f2a", "2001:db8:a4f8:112a::", prefixLength, day1), + new TestIpDataHolder("2001:db8:a4f8:112a:7044:1f01:0:44f2", "2001:db8:a4f8:112a::", prefixLength, day1), + new TestIpDataHolder("2001:db8:a4ff:112a::7002:7ff2", "2001:db8:a4ff:112a::", prefixLength, day2), + new TestIpDataHolder("3007:db81:4b11:234f:1212:0:1:3", "3007:db81:4b11:234f::", prefixLength, day2), + new TestIpDataHolder("3007:db81:4b11:234f:7770:12f6:0:30", "3007:db81:4b11:234f::", prefixLength, day1) + ); + + final Set expectedBucket1Subnets = ipAddresses.stream() + .filter(testIpDataHolder -> testIpDataHolder.getTime() == day1) + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set expectedBucket2Subnets = ipAddresses.stream() + .filter(testIpDataHolder -> testIpDataHolder.getTime() == day2) + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + + // WHEN + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (final TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + List.of( + new SortedDocValuesField(ipv4FieldName, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress()))), + new SortedNumericDocValuesField(datetimeFieldName, ipDataHolder.getTime()) + ) + ); + } + }, agg -> { + final InternalDateHistogram dateHistogram = (InternalDateHistogram) agg; + final List buckets = dateHistogram.getBuckets(); + assertEquals(2, buckets.size()); + + final InternalDateHistogram.Bucket day1Bucket = buckets.stream() + .filter(bucket -> bucket.getKey().equals(Instant.ofEpochMilli(day1).atZone(ZoneOffset.UTC))) + .findAny() + .orElse(null); + final InternalDateHistogram.Bucket day2Bucket = buckets.stream() + .filter(bucket -> bucket.getKey().equals(Instant.ofEpochMilli(day2).atZone(ZoneOffset.UTC))) + .findAny() + .orElse(null); + final InternalIpPrefix ipPrefix1 = Objects.requireNonNull(day1Bucket).getAggregations().get(ipPrefixAggregationName); + final InternalIpPrefix ipPrefix2 = Objects.requireNonNull(day2Bucket).getAggregations().get(ipPrefixAggregationName); + assertNotNull(ipPrefix1); + assertNotNull(ipPrefix2); + assertEquals(expectedBucket1Subnets.size(), ipPrefix1.getBuckets().size()); + assertEquals(expectedBucket2Subnets.size(), ipPrefix2.getBuckets().size()); + + final Set bucket1Subnets = ipPrefix1.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set bucket2Subnets = ipPrefix2.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + assertTrue(bucket1Subnets.containsAll(expectedBucket1Subnets)); + assertTrue(bucket2Subnets.containsAll(expectedBucket2Subnets)); + assertTrue(expectedBucket1Subnets.containsAll(bucket1Subnets)); + assertTrue(expectedBucket2Subnets.containsAll(bucket2Subnets)); + }, fieldTypes); + } + + public void testIpPrefixSubAggregations() throws IOException { + // GIVEN + final int topPrefixLength = 16; + final int subPrefixLength = 24; + final String ipv4FieldName = "ipv4"; + final String topIpPrefixAggregation = "top_ip_prefix"; + final String subIpPrefixAggregation = "sub_ip_prefix"; + final AggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder(topIpPrefixAggregation).field(ipv4FieldName) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(topPrefixLength) + .subAggregation( + new IpPrefixAggregationBuilder(subIpPrefixAggregation).field(ipv4FieldName) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(subPrefixLength) + ); + final IpFieldMapper.IpFieldType ipFieldType = new IpFieldMapper.IpFieldType(ipv4FieldName); + final MappedFieldType[] fieldTypes = { ipFieldType }; + + final String FIRST_SUBNET = "192.168.0.0"; + final String SECOND_SUBNET = "192.169.0.0"; + final List ipAddresses = List.of( + new TestIpDataHolder("192.168.1.12", FIRST_SUBNET, topPrefixLength, defaultTime()), + new TestIpDataHolder("192.168.10.12", FIRST_SUBNET, topPrefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.117", FIRST_SUBNET, topPrefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.27", FIRST_SUBNET, topPrefixLength, defaultTime()), + new TestIpDataHolder("192.169.1.18", SECOND_SUBNET, topPrefixLength, defaultTime()), + new TestIpDataHolder("192.168.2.129", FIRST_SUBNET, topPrefixLength, defaultTime()), + new TestIpDataHolder("192.169.2.49", SECOND_SUBNET, topPrefixLength, defaultTime()), + new TestIpDataHolder("192.169.1.201", SECOND_SUBNET, topPrefixLength, defaultTime()) + ); + + // WHEN + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (final TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + List.of(new SortedDocValuesField(ipv4FieldName, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) + ); + } + }, agg -> { + final InternalIpPrefix topIpPrefix = (InternalIpPrefix) agg; + final List buckets = topIpPrefix.getBuckets(); + assertEquals(2, buckets.size()); + + final InternalIpPrefix.Bucket firstSubnetBucket = topIpPrefix.getBuckets() + .stream() + .filter(bucket -> FIRST_SUBNET.equals(bucket.getKeyAsString())) + .findAny() + .orElse(null); + final InternalIpPrefix.Bucket secondSubnetBucket = topIpPrefix.getBuckets() + .stream() + .filter(bucket -> SECOND_SUBNET.equals(bucket.getKeyAsString())) + .findAny() + .orElse(null); + assertNotNull(firstSubnetBucket); + assertNotNull(secondSubnetBucket); + assertEquals(5, firstSubnetBucket.getDocCount()); + assertEquals(3, secondSubnetBucket.getDocCount()); + + final InternalIpPrefix firstBucketSubAggregation = firstSubnetBucket.getAggregations().get(subIpPrefixAggregation); + final InternalIpPrefix secondBucketSubAggregation = secondSubnetBucket.getAggregations().get(subIpPrefixAggregation); + final Set firstSubnetNestedSubnets = firstBucketSubAggregation.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set secondSubnetNestedSubnets = secondBucketSubAggregation.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + final List expectedFirstSubnetNestedSubnets = List.of("192.168.1.0", "192.168.2.0", "192.168.10.0"); + final List expectedSecondSubnetNestedSUbnets = List.of("192.169.1.0", "192.169.2.0"); + assertTrue(firstSubnetNestedSubnets.containsAll(expectedFirstSubnetNestedSubnets)); + assertTrue(expectedFirstSubnetNestedSubnets.containsAll(firstSubnetNestedSubnets)); + assertTrue(secondSubnetNestedSubnets.containsAll(expectedSecondSubnetNestedSUbnets)); + assertTrue(expectedSecondSubnetNestedSUbnets.containsAll(secondSubnetNestedSubnets)); + + }, fieldTypes); + } + + public void testIpv4AppendPrefixLength() throws IOException { + // GIVEN + final int prefixLength = 16; + final String field = "ipv4"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(true) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = List.of( + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.117", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.10.27", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.169.0.88", "192.169.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.19.0.44", "10.19.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.122.2.67", "10.122.0.0", prefixLength, defaultTime()) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .map(appendPrefixLength(prefixLength)) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .map(appendPrefixLength(prefixLength)) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertFalse(bucket.isIpv6()); + assertTrue(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testIpv6AppendPrefixLength() throws IOException { + // GIVEN + final int prefixLength = 64; + final String field = "ipv6"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(true) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = List.of( + new TestIpDataHolder("2001:db8:a4f8:112a:6001:0:12:7f2a", "2001:db8:a4f8:112a::", prefixLength, defaultTime()), + new TestIpDataHolder("2001:db8:a4f8:112a:7044:1f01:0:44f2", "2001:db8:a4f8:112a::", prefixLength, defaultTime()), + new TestIpDataHolder("2001:db8:a4ff:112a::7002:7ff2", "2001:db8:a4ff:112a::", prefixLength, defaultTime()), + new TestIpDataHolder("3007:db81:4b11:234f:1212:0:1:3", "3007:db81:4b11:234f::", prefixLength, defaultTime()), + new TestIpDataHolder("3007:db81:4b11:234f:7770:12f6:0:30", "3007:db81:4b11:234f::", prefixLength, defaultTime()) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .map(appendPrefixLength(prefixLength)) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .map(appendPrefixLength(prefixLength)) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertTrue(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testMinDocCount() throws IOException { + // GIVEN + final int prefixLength = 16; + final String field = "ipv4"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(2) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = List.of( + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.117", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.10.27", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.169.0.88", "192.169.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.19.0.44", "10.19.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.122.2.67", "10.122.0.0", prefixLength, defaultTime()) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = Set.of("192.168.0.0"); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertFalse(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testAggregationWithQueryFilter() throws IOException { + // GIVEN + final int prefixLength = 16; + final String field = "ipv4"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder("ip_prefix").field(field) + .isIpv6(false) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength); + final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); + final List ipAddresses = List.of( + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.12", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.1.117", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.168.10.27", "192.168.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("192.169.0.88", "192.169.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.19.0.44", "10.19.0.0", prefixLength, defaultTime()), + new TestIpDataHolder("10.122.2.67", "10.122.0.0", prefixLength, defaultTime()) + ); + final Query query = InetAddressPoint.newRangeQuery( + field, + InetAddresses.forString("192.168.0.0"), + InetAddressPoint.nextDown(InetAddresses.forString("192.169.0.0")) + ); + + // WHEN + testAggregation(aggregationBuilder, query, iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + List.of( + new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress()))), + new InetAddressPoint(field, ipDataHolder.getIpAddress()) + ) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .filter(subnet -> subnet.startsWith("192.168.")) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertFalse(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + }, fieldType); + } + + public void testMetricAggregation() throws IOException { + // GIVEN + final int prefixLength = 64; + final String ipField = "ipv6"; + final String timeField = "time"; + final String topAggregationName = "ip_prefix"; + final String subAggregationName = "total_time"; + final IpPrefixAggregationBuilder aggregationBuilder = new IpPrefixAggregationBuilder(topAggregationName).field(ipField) + .isIpv6(true) + .keyed(randomBoolean()) + .appendPrefixLength(false) + .minDocCount(0) + .prefixLength(prefixLength) + .subAggregation(new SumAggregationBuilder(subAggregationName).field(timeField)); + final MappedFieldType[] fieldTypes = { + new IpFieldMapper.IpFieldType(ipField), + new NumberFieldMapper.NumberFieldType(timeField, NumberFieldMapper.NumberType.LONG) }; + final List ipAddresses = List.of( + new TestIpDataHolder("2001:db8:a4f8:112a:6001:0:12:7f2a", "2001:db8:a4f8:112a::", prefixLength, 100), + new TestIpDataHolder("2001:db8:a4f8:112a:7044:1f01:0:44f2", "2001:db8:a4f8:112a::", prefixLength, 110), + new TestIpDataHolder("2001:db8:a4ff:112a::7002:7ff2", "2001:db8:a4ff:112a::", prefixLength, 200), + new TestIpDataHolder("3007:db81:4b11:234f:1212:0:1:3", "3007:db81:4b11:234f::", prefixLength, 170), + new TestIpDataHolder("3007:db81:4b11:234f:7770:12f6:0:30", "3007:db81:4b11:234f::", prefixLength, 130) + ); + + // WHEN + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (TestIpDataHolder ipDataHolder : ipAddresses) { + iw.addDocument( + List.of( + new SortedDocValuesField(ipField, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress()))), + new NumericDocValuesField(timeField, ipDataHolder.getTime()) + ) + ); + } + }, ipPrefix -> { + final Set expectedSubnets = ipAddresses.stream() + .map(TestIpDataHolder::getSubnetAsString) + .collect(Collectors.toUnmodifiableSet()); + final Set ipAddressesAsString = ipPrefix.getBuckets() + .stream() + .map(InternalIpPrefix.Bucket::getKeyAsString) + .collect(Collectors.toUnmodifiableSet()); + + // THEN + ipPrefix.getBuckets().forEach(bucket -> { + assertTrue(bucket.isIpv6()); + assertFalse(bucket.appendPrefixLength()); + assertEquals(prefixLength, bucket.getPrefixLength()); + }); + assertFalse(ipPrefix.getBuckets().isEmpty()); + assertEquals(expectedSubnets.size(), ipPrefix.getBuckets().size()); + assertTrue(ipAddressesAsString.containsAll(expectedSubnets)); + assertTrue(expectedSubnets.containsAll(ipAddressesAsString)); + + assertEquals(210, ((InternalSum) ipPrefix.getBuckets().get(0).getAggregations().get(subAggregationName)).getValue(), 0); + assertEquals(200, ((InternalSum) ipPrefix.getBuckets().get(1).getAggregations().get(subAggregationName)).getValue(), 0); + assertEquals(300, ((InternalSum) ipPrefix.getBuckets().get(2).getAggregations().get(subAggregationName)).getValue(), 0); + }, fieldTypes); + } + + private Function appendPrefixLength(int prefixLength) { + return subnetAddress -> subnetAddress + "/" + prefixLength; + } + + private long defaultTime() { + return randomLongBetween(0, Long.MAX_VALUE); + } + + private void testAggregation( + AggregationBuilder aggregationBuilder, + Query query, + CheckedConsumer buildIndex, + Consumer verify, + MappedFieldType... fieldTypes + ) throws IOException { + testCase(aggregationBuilder, query, buildIndex, verify, fieldTypes); + } +} From 77373e9e98dd1a1303e9ac9f4397abc456d52a69 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 11 Jan 2022 10:53:06 +0100 Subject: [PATCH 03/36] fix: reuse method to extract the netmask from the prefix length --- .../bucket/range/InternalIpPrefix.java | 20 +-------- .../range/IpPrefixAggregationBuilder.java | 42 ++++++++++--------- .../bucket/range/IpPrefixAggregator.java | 36 ++++++++-------- 3 files changed, 42 insertions(+), 56 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java index 041d8e22bb501..e1dd7b27ac317 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.BitSet; import java.util.Collections; import java.util.List; import java.util.Map; @@ -100,24 +99,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } private static BytesRef netmask(int prefixLength) { - final BitSet bs = new BitSet(32); - bs.set(0, 32, false); - bs.set(32 - prefixLength, 32, true); - return new BytesRef(toBigEndian(toIpv4ByteArray(bs.toByteArray()))); - } - - private static byte[] toIpv4ByteArray(byte[] array) { - byte[] netmask = new byte[4]; - System.arraycopy(array, 0, netmask, 0, array.length); - return netmask; - } - - public static byte[] toBigEndian(byte[] array) { - byte[] result = new byte[array.length]; - for (int i = 0; i < array.length; i++) { - result[array.length - i - 1] = array[i]; - } - return result; + return IpPrefixAggregationBuilder.extractNetmask(prefixLength, false); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java index fbda1346a4ffb..9f2494d084752 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.Arrays; import java.util.Map; import java.util.Objects; @@ -196,25 +197,11 @@ protected ValuesSourceAggregatorFactory innerBuild( ); } - byte[] subnet = extractSubnet(prefixLength, isIpv6); - if (subnet == null) { - throw new IllegalArgumentException( - "[" - + PREFIX_LENGTH_FIELD.getPreferredName() - + "] must be in range [" - + MIN_PREFIX_LENGTH - + ", " - + IPV4_MAX_PREFIX_LENGTH - + "] for aggregation [" - + this.getName() - + "]" - ); - } IpPrefixAggregator.IpPrefix ipPrefix = new IpPrefixAggregator.IpPrefix( isIpv6, prefixLength, appendPrefixLength, - new BytesRef(subnet) + extractNetmask(prefixLength, isIpv6) ); return new IpPrefixAggregatorFactory( @@ -231,9 +218,26 @@ protected ValuesSourceAggregatorFactory innerBuild( ); } - private static byte[] extractSubnet(int prefixLength, boolean isIpv6) { + /** + * @param prefixLength the network prefix length which defines the size of the network. + * @param isIpv6 true for an IPv6 netmask, false for an IPv4 netmask. + * + * @return a 16-bytes representation of the subnet with 1s identifying the network + * part and 0s identifying the host part. + * + * @throws IllegalArgumentException if prefixLength is not in range [0, 128] for an IPv6 + * network, or is not in range [0, 32] for an IPv4 network. + */ + public static BytesRef extractNetmask(int prefixLength, boolean isIpv6) { if (prefixLength < 0 || (!isIpv6 && prefixLength > 32) || (isIpv6 && prefixLength > 128)) { - return null; + throw new IllegalArgumentException( + "[" + PREFIX_LENGTH_FIELD.getPreferredName() + + "] must be in range [" + + MIN_PREFIX_LENGTH + + ", " + + (isIpv6 ? IPV6_MAX_PREFIX_LENGTH : IPV4_MAX_PREFIX_LENGTH) + + "]" + ); } byte[] ipv4Address = { 0, 0, 0, 0 }; @@ -254,9 +258,9 @@ private static byte[] extractSubnet(int prefixLength, boolean isIpv6) { } try { - return InetAddress.getByAddress(ipAddress).getAddress(); + return new BytesRef(InetAddress.getByAddress(ipAddress).getAddress()); } catch (UnknownHostException e) { - return null; + throw new IllegalArgumentException("Unable to get the ip address for [" + Arrays.toString(ipAddress) + "]", e); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java index bbf06cd6018e6..3608a2b14f45a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java @@ -42,13 +42,13 @@ public static class IpPrefix { final boolean isIpv6; final int prefixLength; final boolean appendPrefixLength; - final BytesRef mask; + final BytesRef netmask; - public IpPrefix(boolean isIpv6, int prefixLength, boolean appendPrefixLength, BytesRef mask) { + public IpPrefix(boolean isIpv6, int prefixLength, boolean appendPrefixLength, BytesRef netmask) { this.isIpv6 = isIpv6; this.prefixLength = prefixLength; this.appendPrefixLength = appendPrefixLength; - this.mask = mask; + this.netmask = netmask; } public boolean isIpv6() { @@ -63,8 +63,8 @@ public boolean appendPrefixLength() { return appendPrefixLength; } - public BytesRef getMask() { - return mask; + public BytesRef getNetmask() { + return netmask; } @Override @@ -75,12 +75,12 @@ public boolean equals(Object o) { return isIpv6 == ipPrefix.isIpv6 && prefixLength == ipPrefix.prefixLength && appendPrefixLength == ipPrefix.appendPrefixLength - && Objects.equals(mask, ipPrefix.mask); + && Objects.equals(netmask, ipPrefix.netmask); } @Override public int hashCode() { - return Objects.hash(isIpv6, prefixLength, appendPrefixLength, mask); + return Objects.hash(isIpv6, prefixLength, appendPrefixLength, netmask); } } @@ -141,12 +141,12 @@ public void collect(int doc, long owningBucketOrd) throws IOException { for (int i = 0; i < valuesCount; ++i) { BytesRef value = values.nextValue(); byte[] ipAddress = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); - byte[] mask = Arrays.copyOfRange( - ipPrefix.mask.bytes, - ipPrefix.mask.offset, - ipPrefix.mask.offset + ipPrefix.mask.length + byte[] netmask = Arrays.copyOfRange( + ipPrefix.netmask.bytes, + ipPrefix.netmask.offset, + ipPrefix.netmask.offset + ipPrefix.netmask.length ); - byte[] subnet = maskIpAddress(ipAddress, mask); + byte[] subnet = maskIpAddress(ipAddress, netmask); if (Arrays.equals(subnet, previousSubnet)) { continue; } @@ -162,19 +162,19 @@ public void collect(int doc, long owningBucketOrd) throws IOException { } } - private byte[] maskIpAddress(byte[] ipAddress, byte[] subnetMask) { + private byte[] maskIpAddress(byte[] ipAddress, byte[] netmask) { // NOTE: ip addresses are always encoded to 16 bytes by IpFieldMapper if (ipAddress.length != 16) { throw new IllegalArgumentException("Invalid length for ip address [" + ipAddress.length + "]"); } - if (subnetMask.length == 4) { - return mask(Arrays.copyOfRange(ipAddress, 12, 16), subnetMask); + if (netmask.length == 4) { + return mask(Arrays.copyOfRange(ipAddress, 12, 16), netmask); } - if (subnetMask.length == 16) { - return mask(ipAddress, subnetMask); + if (netmask.length == 16) { + return mask(ipAddress, netmask); } - throw new IllegalArgumentException("Invalid length for subnet mask [" + subnetMask.length + "]"); + throw new IllegalArgumentException("Invalid length for netmask [" + netmask.length + "]"); } private byte[] mask(byte[] ipAddress, byte[] subnetMask) { From f0e4b8acc65a67b1d75fc3eb32afed6f67a8148e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 11 Jan 2022 11:04:50 +0100 Subject: [PATCH 04/36] docs: add some javadoc --- .../bucket/range/InternalIpPrefix.java | 6 ++++++ .../search/aggregations/bucket/range/IpPrefix.java | 8 +++++++- .../bucket/range/IpPrefixAggregationBuilder.java | 14 ++++++++++++++ .../bucket/range/IpPrefixAggregator.java | 3 +++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java index e1dd7b27ac317..08a4f51827db3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java @@ -64,6 +64,9 @@ public Bucket( this.aggregations = aggregations; } + /** + * Read from a stream. + */ public Bucket(StreamInput in, DocValueFormat format, boolean keyed) throws IOException { this.format = format; this.keyed = keyed; @@ -192,6 +195,9 @@ public InternalIpPrefix( this.buckets = buckets; } + /** + * Stream from a stream. + */ public InternalIpPrefix(StreamInput in) throws IOException { super(in); format = in.readNamedWriteable(DocValueFormat.class); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java index ac89a787404ea..9da1643dbfe52 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java @@ -6,6 +6,9 @@ * Side Public License, v 1. */ +/** + * A {@code ip prefix} aggregation. Defines multiple buckets, each representing a subnet. + */ package org.elasticsearch.search.aggregations.bucket.range; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; @@ -14,12 +17,15 @@ public interface IpPrefix extends MultiBucketsAggregation { + /** + * A bucket in the aggregation where documents fall in + */ interface Bucket extends MultiBucketsAggregation.Bucket { } /** - * Return the buckets of this range aggregation. + * @return The buckets of this aggregation (each bucket representing a subnet) */ @Override List getBuckets(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java index 9f2494d084752..efb80649308ec 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java @@ -32,6 +32,9 @@ import java.util.Map; import java.util.Objects; +/** + * A builder for IP prefix aggregations. This builder can operate with both IPv4 and IPv6 fields. + */ public class IpPrefixAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "ip_prefix"; public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>( @@ -62,6 +65,7 @@ public class IpPrefixAggregationBuilder extends ValuesSourceAggregationBuilder Date: Tue, 11 Jan 2022 11:24:40 +0100 Subject: [PATCH 05/36] fix: code format violations --- .../bucket/range/InternalIpPrefix.java | 6 ++--- .../aggregations/bucket/range/IpPrefix.java | 6 ++--- .../range/IpPrefixAggregationBuilder.java | 9 ++++---- .../bucket/range/IpPrefixAggregator.java | 2 +- .../bucket/range/IpPrefixAggregatorTests.java | 23 +++++++++++++++---- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java index 08a4f51827db3..e888ba45b5b8a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java @@ -90,7 +90,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(CommonFields.KEY.getPreferredName(), key); } - if (!isIpv6) { + if (isIpv6 == false) { builder.field("netmask", DocValueFormat.IP.format(netmask(prefixLength))); } builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); @@ -249,7 +249,7 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent top = pq.top(); - if (!top.current().key.equals(value)) { + if (top.current().key.equals(value) == false) { final Bucket reduced = reduceBucket(currentBuckets, reduceContext); if (reduced.getDocCount() >= minDocCount) { reducedBuckets.add(reduced); @@ -338,7 +338,7 @@ public List getBuckets() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; + if (super.equals(o) == false) return false; InternalIpPrefix that = (InternalIpPrefix) o; return minDocCount == that.minDocCount && Objects.equals(format, that.format) && Objects.equals(buckets, that.buckets); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java index 9da1643dbfe52..0f4e9a2d599d2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java @@ -6,15 +6,15 @@ * Side Public License, v 1. */ -/** - * A {@code ip prefix} aggregation. Defines multiple buckets, each representing a subnet. - */ package org.elasticsearch.search.aggregations.bucket.range; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import java.util.List; +/** + * A {@code ip prefix} aggregation. Defines multiple buckets, each representing a subnet. + */ public interface IpPrefix extends MultiBucketsAggregation { /** diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java index efb80649308ec..46720e53d3ef1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java @@ -197,7 +197,7 @@ protected ValuesSourceAggregatorFactory innerBuild( ); } - if (!isIpv6 && prefixLength > IPV4_MAX_PREFIX_LENGTH) { + if (isIpv6 == false && prefixLength > IPV4_MAX_PREFIX_LENGTH) { throw new IllegalArgumentException( "[" + PREFIX_LENGTH_FIELD.getPreferredName() @@ -243,9 +243,10 @@ protected ValuesSourceAggregatorFactory innerBuild( * network, or is not in range [0, 32] for an IPv4 network. */ public static BytesRef extractNetmask(int prefixLength, boolean isIpv6) { - if (prefixLength < 0 || (!isIpv6 && prefixLength > 32) || (isIpv6 && prefixLength > 128)) { + if (prefixLength < 0 || (isIpv6 == false && prefixLength > 32) || (isIpv6 && prefixLength > 128)) { throw new IllegalArgumentException( - "[" + PREFIX_LENGTH_FIELD.getPreferredName() + "[" + + PREFIX_LENGTH_FIELD.getPreferredName() + "] must be in range [" + MIN_PREFIX_LENGTH + ", " @@ -292,7 +293,7 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; + if (super.equals(o) == false) return false; IpPrefixAggregationBuilder that = (IpPrefixAggregationBuilder) o; return minDocCount == that.minDocCount && prefixLength == that.prefixLength && isIpv6 == that.isIpv6; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java index 6ed07a5060e4b..8b12e2d5fff49 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java @@ -128,7 +128,7 @@ private class IpPrefixLeafCollector extends LeafBucketCollectorBase { private final LeafBucketCollector sub; private final SortedBinaryDocValues values; - public IpPrefixLeafCollector(LeafBucketCollector sub, SortedBinaryDocValues values, IpPrefix ipPrefix) { + IpPrefixLeafCollector(LeafBucketCollector sub, SortedBinaryDocValues values, IpPrefix ipPrefix) { super(sub, values); this.sub = sub; this.values = values; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java index 9cecf1b57c908..5f9fb3bfedf1e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java @@ -54,7 +54,7 @@ private static final class TestIpDataHolder { private final int prefixLength; private final long time; - public TestIpDataHolder(final String ipAddressAsString, final String subnetAsString, final int prefixLength, final long time) { + TestIpDataHolder(final String ipAddressAsString, final String subnetAsString, final int prefixLength, final long time) { this.ipAddressAsString = ipAddressAsString; this.ipAddress = InetAddresses.forString(ipAddressAsString); this.subnetAsString = subnetAsString; @@ -107,8 +107,22 @@ public int hashCode() { @Override public String toString() { - return "TestIpDataHolder{ipAddressAsString='%s', ipAddress=%s, subnetAsString='%s', subnet=%s, prefixLength=%d, time=%d}" - .formatted(ipAddressAsString, ipAddress, subnetAsString, subnet, prefixLength, time); + return "TestIpDataHolder{" + + "ipAddressAsString='" + + ipAddressAsString + + '\'' + + ", ipAddress=" + + ipAddress + + ", subnetAsString='" + + subnetAsString + + '\'' + + ", subnet=" + + subnet + + ", prefixLength=" + + prefixLength + + ", time=" + + time + + '}'; } } @@ -126,8 +140,7 @@ public void testEmptyDocument() throws IOException { final List ipAddresses = Collections.emptyList(); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { - }, ipPrefix -> { + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> {}, ipPrefix -> { final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); From 3506bbaf148fc1f707ffb0648bbb09b205d0d6e8 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 11 Jan 2022 11:56:06 +0100 Subject: [PATCH 06/36] fix: check number of buckets to collect --- .../aggregations/bucket/range/IpPrefixAggregator.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java index 8b12e2d5fff49..13088929636e2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java @@ -199,6 +199,11 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I bucketsInOrd[ordIdx] = (int) bucketCount; totalOrdsToCollect += bucketCount; } + if (totalOrdsToCollect > Integer.MAX_VALUE) { + throw new AggregationExecutionException( + "Can't collect more than [" + Integer.MAX_VALUE + "] buckets but attempted [" + totalOrdsToCollect + "]" + ); + } long[] bucketOrdsToCollect = new long[(int) totalOrdsToCollect]; int b = 0; @@ -208,6 +213,7 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I bucketOrdsToCollect[b++] = ordsEnum.ord(); } } + InternalAggregations[] subAggregationResults = buildSubAggsForBuckets(bucketOrdsToCollect); InternalAggregation[] results = new InternalAggregation[owningBucketOrds.length]; b = 0; From 3832cb55839d445693b3ff6d75e127dc3504b12f Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 11 Jan 2022 12:40:02 +0100 Subject: [PATCH 07/36] fix: extract ther ipv6 address value --- .../aggregations/bucket/range/IpPrefixAggregatorTests.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java index 5f9fb3bfedf1e..92a2ab1877f51 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java @@ -433,6 +433,7 @@ public void testAggregateOnIpv4Field() throws IOException { new TestIpDataHolder("10.19.0.44", "10.19.0.0", prefixLength, defaultTime()), new TestIpDataHolder("10.122.2.67", "10.122.0.0", prefixLength, defaultTime()) ); + final String ipv6Value = "2001:db8:a4f8:112a:6001:0:12:7f2a"; // WHEN testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { @@ -440,10 +441,7 @@ public void testAggregateOnIpv4Field() throws IOException { iw.addDocument( List.of( new SortedDocValuesField(ipv4FieldName, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress()))), - new SortedDocValuesField( - ipv6FieldName, - new BytesRef(InetAddressPoint.encode(InetAddresses.forString("2001:db8:a4f8:112a:6001:0:12:7f2a"))) - ) + new SortedDocValuesField(ipv6FieldName, new BytesRef(InetAddressPoint.encode(InetAddresses.forString(ipv6Value)))) ) ); } From 228c264ab68f9980777de8b21af2c59913a38900 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 11 Jan 2022 14:07:48 +0100 Subject: [PATCH 08/36] fix: add ip_prefix to the unsupported transform aggregations --- .../xpack/transform/transforms/pivot/TransformAggregations.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java index 95ece2ac49703..c97eb517841a4 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java @@ -65,6 +65,7 @@ public final class TransformAggregations { "geotile_grid", "global", "histogram", + "ip_prefix", "ip_range", "matrix_stats", "nested", From 48c9d03d2fc59c28ab30dd838633d32c17eb32e7 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Wed, 12 Jan 2022 20:15:46 +0100 Subject: [PATCH 09/36] test: include tests with incorrect is_ipv6 value --- .../test/search.aggregation/450_ip_prefix.yml | 59 ++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml index daba9ce5e4aaa..a404524e8f10c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml @@ -71,6 +71,34 @@ setup: - match: { aggregations.ip_prefix.buckets.1.prefix_len: 24 } - match: { aggregations.ip_prefix.buckets.1.netmask: "255.255.255.0" } +--- +# NOTE: here prefix_len = 24 which means the netmask 255.255.0.0 will be applied to the +# high 16 bits of a field which is an IPv4 address encoded on 16 bytes. As a result the +# network part will just 0s. +"IPv4 prefix with incorrect is_ipv6": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ipv4" + is_ipv6: true + prefix_len: 24 + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.ip_prefix.buckets: 1 } + - match: { aggregations.ip_prefix.buckets.0.key: "::" } + - match: { aggregations.ip_prefix.buckets.0.doc_count: 10 } + - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.ip_prefix.buckets.0.prefix_len: 24 } + --- "IPv4 short prefix": - skip: @@ -124,7 +152,6 @@ setup: is_ipv6: true prefix_len: 64 - - match: { hits.total.value: 10 } - match: { hits.total.relation: "eq" } - length: { aggregations.ip_prefix.buckets: 2 } @@ -139,6 +166,36 @@ setup: - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } - match: { aggregations.ip_prefix.buckets.1.netmask: null } +--- +# NOTE: here prefix_len = 16 which means the netmask will be applied to the second +# group of 2 bytes starting from the right (i.e. for "2001:db8:a4f8:112a:6001:0:12:7f10" +# it will be the 2 bytes whose value is set to 12 hexadecimal) which results to 18 decimal, +# with everything else being 0s. +"IPv6 prefix with incorrect is_ipv6": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ipv6" + is_ipv6: false + prefix_len: 16 + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.ip_prefix.buckets: 1 } + - match: { aggregations.ip_prefix.buckets.0.key: "0.18.0.0" } + - match: { aggregations.ip_prefix.buckets.0.doc_count: 10 } + - is_false: aggregations.ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.ip_prefix.buckets.0.prefix_len: 16 } + - match: { aggregations.ip_prefix.buckets.0.netmask: "255.255.0.0" } + --- "Invalid IPv4 prefix": - skip: From b3403e9ca8441a0a2e71992aa24eee94d58e12e0 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 13 Jan 2022 09:46:16 +0100 Subject: [PATCH 10/36] fix: incorrect netmask --- .../rest-api-spec/test/search.aggregation/450_ip_prefix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml index a404524e8f10c..0f942b99025da 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml @@ -72,8 +72,8 @@ setup: - match: { aggregations.ip_prefix.buckets.1.netmask: "255.255.255.0" } --- -# NOTE: here prefix_len = 24 which means the netmask 255.255.0.0 will be applied to the -# high 16 bits of a field which is an IPv4 address encoded on 16 bytes. As a result the +# NOTE: here prefix_len = 24 which means the netmask 255.255.255.0 will be applied to the +# high 24 bits of a field which is an IPv4 address encoded on 16 bytes. As a result the # network part will just 0s. "IPv4 prefix with incorrect is_ipv6": - skip: From 58297d5bbb7990477e30a23c87fa94bc571c9db9 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 13 Jan 2022 10:21:55 +0100 Subject: [PATCH 11/36] fix: move IpPrefix aggregator to prefix package --- .../src/main/java/org/elasticsearch/search/SearchModule.java | 4 ++-- .../bucket/{range => prefix}/InternalIpPrefix.java | 2 +- .../aggregations/bucket/{range => prefix}/IpPrefix.java | 2 +- .../bucket/{range => prefix}/IpPrefixAggregationBuilder.java | 2 +- .../bucket/{range => prefix}/IpPrefixAggregationSupplier.java | 2 +- .../bucket/{range => prefix}/IpPrefixAggregator.java | 2 +- .../bucket/{range => prefix}/IpPrefixAggregatorFactory.java | 2 +- .../search/aggregations/bucket/IpPrefixTests.java | 2 +- .../bucket/{range => prefix}/IpPrefixAggregatorTests.java | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/{range => prefix}/InternalIpPrefix.java (99%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/{range => prefix}/IpPrefix.java (93%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/{range => prefix}/IpPrefixAggregationBuilder.java (99%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/{range => prefix}/IpPrefixAggregationSupplier.java (95%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/{range => prefix}/IpPrefixAggregator.java (99%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/{range => prefix}/IpPrefixAggregatorFactory.java (98%) rename server/src/test/java/org/elasticsearch/search/aggregations/bucket/{range => prefix}/IpPrefixAggregatorTests.java (99%) diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 3a37632bde262..6b6fa122127e3 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -123,9 +123,9 @@ import org.elasticsearch.search.aggregations.bucket.range.InternalBinaryRange; import org.elasticsearch.search.aggregations.bucket.range.InternalDateRange; import org.elasticsearch.search.aggregations.bucket.range.InternalGeoDistance; -import org.elasticsearch.search.aggregations.bucket.range.InternalIpPrefix; +import org.elasticsearch.search.aggregations.bucket.prefix.InternalIpPrefix; import org.elasticsearch.search.aggregations.bucket.range.InternalRange; -import org.elasticsearch.search.aggregations.bucket.range.IpPrefixAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.prefix.IpPrefixAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.IpRangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefix.java similarity index 99% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefix.java index e888ba45b5b8a..a99c78d7150a9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalIpPrefix.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefix.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.bucket.range; +package org.elasticsearch.search.aggregations.bucket.prefix; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.PriorityQueue; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefix.java similarity index 93% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefix.java index 0f4e9a2d599d2..b4ed37569565d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefix.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefix.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.bucket.range; +package org.elasticsearch.search.aggregations.bucket.prefix; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java similarity index 99% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java index 46720e53d3ef1..78da252f74195 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.bucket.range; +package org.elasticsearch.search.aggregations.bucket.prefix; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java similarity index 95% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java index 33907b3bc982a..2634f1da1ace0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregationSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.bucket.range; +package org.elasticsearch.search.aggregations.bucket.prefix; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java similarity index 99% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java index 13088929636e2..6b77900f5aa5e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.bucket.range; +package org.elasticsearch.search.aggregations.bucket.prefix; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java similarity index 98% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java index ea41835fbdcd1..83bef53f8bf2f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.bucket.range; +package org.elasticsearch.search.aggregations.bucket.prefix; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java index b5943cbd21093..654f39f423bfa 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java @@ -9,7 +9,7 @@ package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.search.aggregations.BaseAggregationTestCase; -import org.elasticsearch.search.aggregations.bucket.range.IpPrefixAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.prefix.IpPrefixAggregationBuilder; import static org.hamcrest.Matchers.startsWith; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java similarity index 99% rename from server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java rename to server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java index 92a2ab1877f51..15628bb3e06b9 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpPrefixAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.bucket.range; +package org.elasticsearch.search.aggregations.bucket.prefix; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.NumericDocValuesField; From 563d337b1d663a5fe2d43b280f0f9550a160f5ad Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 13 Jan 2022 10:38:05 +0100 Subject: [PATCH 12/36] fix: code format violations --- .../src/main/java/org/elasticsearch/search/SearchModule.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 6b6fa122127e3..85c2747d19eeb 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -118,14 +118,14 @@ import org.elasticsearch.search.aggregations.bucket.nested.InternalReverseNested; import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.prefix.InternalIpPrefix; +import org.elasticsearch.search.aggregations.bucket.prefix.IpPrefixAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.GeoDistanceAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.InternalBinaryRange; import org.elasticsearch.search.aggregations.bucket.range.InternalDateRange; import org.elasticsearch.search.aggregations.bucket.range.InternalGeoDistance; -import org.elasticsearch.search.aggregations.bucket.prefix.InternalIpPrefix; import org.elasticsearch.search.aggregations.bucket.range.InternalRange; -import org.elasticsearch.search.aggregations.bucket.prefix.IpPrefixAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.IpRangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder; From c269c66b90aca6942a1f7178b7c2697c229d11c0 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 13 Jan 2022 17:35:12 +0100 Subject: [PATCH 13/36] docs: include documentation for the ip prefix aggregation --- docs/reference/aggregations/bucket.asciidoc | 2 + .../bucket/ipprefix-aggregation.asciidoc | 340 ++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc index dfdaca18e6cfb..88fe92c27f9b3 100644 --- a/docs/reference/aggregations/bucket.asciidoc +++ b/docs/reference/aggregations/bucket.asciidoc @@ -46,6 +46,8 @@ include::bucket/global-aggregation.asciidoc[] include::bucket/histogram-aggregation.asciidoc[] +include::bucket/ipprefix-aggregation.asciidoc[] + include::bucket/iprange-aggregation.asciidoc[] include::bucket/missing-aggregation.asciidoc[] diff --git a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc new file mode 100644 index 0000000000000..3734e5d5c8f31 --- /dev/null +++ b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc @@ -0,0 +1,340 @@ +[[search-aggregations-bucket-ipprefix-aggregation]] +=== IP prefix aggregation +++++ +IP prefix +++++ +IP addresses can be aggregated according to their network or sub-network. IP addresses consists of two groups of bits: the most significant bits which represent the network prefix, and the least significant bits which represent the host. +The network or sub-network is usually identified using a netmask, or a prefix length. The prefix length defines the number of bits representing the network prefix. + +For example consider the following index: +[source,console] +---------------------------------------------- +PUT network-traffic +{ + "mappings": { + "properties": { + "ipv4": { "type": "ip" }, + "ipv6": { "type": "ip" } + } + } +} + +POST /network-traffic/_bulk?refresh +{"index":{"_id":0}} +{"ipv4":"192.168.1.10","ipv6":"2001:db8:a4f8:112a:6001:0:12:7f10"} +{"index":{"_id":1}} +{"ipv4":"192.168.1.12","ipv6":"2001:db8:a4f8:112a:6001:0:12:7f12"} +{"index":{"_id":2}} +{ "ipv4":"192.168.1.33","ipv6":"2001:db8:a4f8:112a:6001:0:12:7f33"} +{"index":{"_id":3}} +{"ipv4":"192.168.1.10","ipv6":"2001:db8:a4f8:112a:6001:0:12:7f10"} +{"index":{"_id":4}} +{"ipv4":"192.168.1.33","ipv6":"2001:db8:a4f8:112a:6001:0:12:7f33"} +{"index":{"_id":5}} +{"ipv4":"192.168.2.41","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f41"} +{"index":{"_id":6}} +{"ipv4":"192.168.2.10","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f10"} +{"index":{"_id":7}} +{"ipv4":"192.168.2.23","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f23"} +{"index":{"_id":8}} +{"ipv4":"192.168.2.41","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f41"} +{"index":{"_id":9}} +{"ipv4":"192.168.2.10","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f10"} +{"index":{"_id":10}} +{"ipv4":"192.168.3.201","ipv6":"2001:db8:a4f8:114f:6001:0:12:7201"} +{"index":{"_id":11}} +{"ipv4":"192.168.3.107","ipv6":"2001:db8:a4f8:114f:6001:0:12:7307"} +---------------------------------------------- +// TESTSETUP + +The following aggregation will aggregate documents in buckets, each identifying a different sub-network. The sub-network is calculated applying a netmask with prefix length of `24` to each IP address in the `ipv4` field: + +[source,console,id=ip-prefix-ipv4-example] +-------------------------------------------------- +GET /network-traffic/_search +{ + "size": 0, + "aggs": { + "ipv4-subnets": { + "ip_prefix": { + "field": "ipv4", + "prefix_len": 24 + } + } + } +} +-------------------------------------------------- +// TEST + +Response: + +[source,console-result] +-------------------------------------------------- +{ + ... + + "aggregations": { + "ipv4-subnets": { + "buckets": [ + { + "key": "192.168.1.0", + "is_ipv6": false, + "doc_count": 5, + "prefix_len": 24, + "netmask": "255.255.255.0" + }, + { + "key": "192.168.2.0", + "is_ipv6": false, + "doc_count": 5, + "prefix_len": 24, + "netmask": "255.255.255.0" + }, + { + "key": "192.168.3.0", + "is_ipv6": false, + "doc_count": 2, + "prefix_len": 24, + "netmask": "255.255.255.0" + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] + +In a similar way we can aggregate IPv6 addresses, making sure we set the `is_ipv6` flag to `true` (see section below): + +[source,console,id=ip-prefix-ipv6-example] +-------------------------------------------------- +GET /network-traffic/_search +{ + "size": 0, + "aggs": { + "ipv6-subnets": { + "ip_prefix": { + "field": "ipv6", + "prefix_len": 64, + "is_ipv6": true + } + } + } +} +-------------------------------------------------- +// TEST + +Response: + +[source,console-result] +-------------------------------------------------- +{ + ... + + "aggregations": { + "ipv6-subnets": { + "buckets": [ + { + "key": "2001:db8:a4f8:112a::", + "is_ipv6": true, + "doc_count": 5, + "prefix_len": 64 + }, + { + "key": "2001:db8:a4f8:112c::", + "is_ipv6": true, + "doc_count": 5, + "prefix_len": 64 + }, + { + "key": "2001:db8:a4f8:114f::", + "is_ipv6": true, + "doc_count": 2, + "prefix_len": 64 + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] + +NOTE: the `netmask` field is not returned in the response when flag `is_ipv6` is set to `true`. + +==== Keyed Response + +Setting the `keyed` flag to `true` will associate a unique IP address key with each bucket and return sub-networks as a hash rather than an array: + +Example: + +[source,console,id=ip-prefix-keyed-example] +-------------------------------------------------- +GET /network-traffic/_search +{ + "size": 0, + "aggs": { + "ipv4-subnets": { + "ip_prefix": { + "field": "ipv4", + "prefix_len": 24, + "keyed": true + } + } + } +} +-------------------------------------------------- +// TEST + +Response: + +[source,console-result] +-------------------------------------------------- +{ + ... + + "aggregations": { + "ipv4-subnets": { + "buckets": { + "192.168.1.0": { + "is_ipv6": false, + "doc_count": 5, + "prefix_len": 24, + "netmask": "255.255.255.0" + }, + "192.168.2.0": { + "is_ipv6": false, + "doc_count": 5, + "prefix_len": 24, + "netmask": "255.255.255.0" + }, + "192.168.3.0": { + "is_ipv6": false, + "doc_count": 2, + "prefix_len": 24, + "netmask": "255.255.255.0" + } + } + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] + +==== Appending the prefix length to the IP address key + +Setting the `append_prefix_len` flag to `true` will return IP address keys catenated with the prefix length of the sub-network: + +Example: + +[source,console,id=ip-prefix-append-prefix-len-example] +-------------------------------------------------- +GET /network-traffic/_search +{ + "size": 0, + "aggs": { + "ipv4-subnets": { + "ip_prefix": { + "field": "ipv4", + "prefix_len": 24, + "append_prefix_len": true + } + } + } +} +-------------------------------------------------- +// TEST + +Response: + +[source,console-result] +-------------------------------------------------- +{ + ... + + "aggregations": { + "ipv4-subnets": { + "buckets": [ + { + "key": "192.168.1.0/24", + "is_ipv6": false, + "doc_count": 5, + "prefix_len": 24, + "netmask": "255.255.255.0" + }, + { + "key": "192.168.2.0/24", + "is_ipv6": false, + "doc_count": 5, + "prefix_len": 24, + "netmask": "255.255.255.0" + }, + { + "key": "192.168.3.0/24", + "is_ipv6": false, + "doc_count": 2, + "prefix_len": 24, + "netmask": "255.255.255.0" + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] + +==== Minimum document count + +It is possible to change the response in such a way that only buckets including a minimum number of documents are returned, using the `min_doc_count` parameter. + +[source,console,id=ip-prefix-min-doc-count-example] +-------------------------------------------------- +GET /network-traffic/_search +{ + "size": 0, + "aggs": { + "ipv4-subnets": { + "ip_prefix": { + "field": "ipv4", + "prefix_len": 24, + "min_doc_count": 3 + } + } + } +} +-------------------------------------------------- +// TEST + +Response: + +[source,console-result] +-------------------------------------------------- +{ + ... + + "aggregations": { + "ipv4-subnets": { + "buckets": [ + { + "key": "192.168.1.0", + "is_ipv6": false, + "doc_count": 5, + "prefix_len": 24, + "netmask": "255.255.255.0" + }, + { + "key": "192.168.2.0", + "is_ipv6": false, + "doc_count": 5, + "prefix_len": 24, + "netmask": "255.255.255.0" + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] + +==== IPv6 vs IPv4 addresses and sub-networks + +Just specifying the `prefix_len` parameter is not enough to know if sub-network aggregation is done on IPv4 or IPv6 addresses. As a result, the `is_ipv6` flag is needed and defaults to `false`. Failing to set it appropriately will result in unpredictable results due to the way the netmask is applied. From 90b5e95755b61fd853a8296e93db1dd29ba8fa45 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 13 Jan 2022 17:41:28 +0100 Subject: [PATCH 14/36] docs: fix sentence using the actual aggregation name --- .../reference/aggregations/bucket/ipprefix-aggregation.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc index 3734e5d5c8f31..b3b8d864cbb51 100644 --- a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc @@ -337,4 +337,4 @@ Response: ==== IPv6 vs IPv4 addresses and sub-networks -Just specifying the `prefix_len` parameter is not enough to know if sub-network aggregation is done on IPv4 or IPv6 addresses. As a result, the `is_ipv6` flag is needed and defaults to `false`. Failing to set it appropriately will result in unpredictable results due to the way the netmask is applied. +Just specifying the `prefix_len` parameter is not enough to know if an IP prefix aggregation is done on IPv4 or IPv6 addresses. As a result, the `is_ipv6` flag is needed and defaults to `false`. Failing to set it appropriately will result in unpredictable results due to the way the netmask is applied. From 3c68ec25a5cf17c0b66e3cab319d93df0c6476cd Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 13 Jan 2022 17:44:59 +0100 Subject: [PATCH 15/36] docs: reduce the number of documents used in the example --- .../bucket/ipprefix-aggregation.asciidoc | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc index b3b8d864cbb51..6988ae5a5f3dd 100644 --- a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc @@ -29,20 +29,14 @@ POST /network-traffic/_bulk?refresh {"index":{"_id":3}} {"ipv4":"192.168.1.10","ipv6":"2001:db8:a4f8:112a:6001:0:12:7f10"} {"index":{"_id":4}} -{"ipv4":"192.168.1.33","ipv6":"2001:db8:a4f8:112a:6001:0:12:7f33"} -{"index":{"_id":5}} {"ipv4":"192.168.2.41","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f41"} -{"index":{"_id":6}} +{"index":{"_id":5}} {"ipv4":"192.168.2.10","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f10"} -{"index":{"_id":7}} +{"index":{"_id":6}} {"ipv4":"192.168.2.23","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f23"} -{"index":{"_id":8}} -{"ipv4":"192.168.2.41","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f41"} -{"index":{"_id":9}} -{"ipv4":"192.168.2.10","ipv6":"2001:db8:a4f8:112c:6001:0:12:7f10"} -{"index":{"_id":10}} +{"index":{"_id":7}} {"ipv4":"192.168.3.201","ipv6":"2001:db8:a4f8:114f:6001:0:12:7201"} -{"index":{"_id":11}} +{"index":{"_id":8}} {"ipv4":"192.168.3.107","ipv6":"2001:db8:a4f8:114f:6001:0:12:7307"} ---------------------------------------------- // TESTSETUP @@ -79,14 +73,14 @@ Response: { "key": "192.168.1.0", "is_ipv6": false, - "doc_count": 5, + "doc_count": 4, "prefix_len": 24, "netmask": "255.255.255.0" }, { "key": "192.168.2.0", "is_ipv6": false, - "doc_count": 5, + "doc_count": 3, "prefix_len": 24, "netmask": "255.255.255.0" }, @@ -137,13 +131,13 @@ Response: { "key": "2001:db8:a4f8:112a::", "is_ipv6": true, - "doc_count": 5, + "doc_count": 4, "prefix_len": 64 }, { "key": "2001:db8:a4f8:112c::", "is_ipv6": true, - "doc_count": 5, + "doc_count": 3, "prefix_len": 64 }, { @@ -197,13 +191,13 @@ Response: "buckets": { "192.168.1.0": { "is_ipv6": false, - "doc_count": 5, + "doc_count": 4, "prefix_len": 24, "netmask": "255.255.255.0" }, "192.168.2.0": { "is_ipv6": false, - "doc_count": 5, + "doc_count": 3, "prefix_len": 24, "netmask": "255.255.255.0" }, @@ -257,14 +251,14 @@ Response: { "key": "192.168.1.0/24", "is_ipv6": false, - "doc_count": 5, + "doc_count": 4, "prefix_len": 24, "netmask": "255.255.255.0" }, { "key": "192.168.2.0/24", "is_ipv6": false, - "doc_count": 5, + "doc_count": 3, "prefix_len": 24, "netmask": "255.255.255.0" }, @@ -317,14 +311,14 @@ Response: { "key": "192.168.1.0", "is_ipv6": false, - "doc_count": 5, + "doc_count": 4, "prefix_len": 24, "netmask": "255.255.255.0" }, { "key": "192.168.2.0", "is_ipv6": false, - "doc_count": 5, + "doc_count": 3, "prefix_len": 24, "netmask": "255.255.255.0" } From 2ca2cb246b7e93f6b808fb202600f26302ff3302 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Fri, 14 Jan 2022 11:41:15 +0100 Subject: [PATCH 16/36] test: add integration tests for mixed IPv4/IPv6 field --- .../test/search.aggregation/450_ip_prefix.yml | 89 ++++++++++++++++--- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml index 0f942b99025da..8d63be257577b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml @@ -11,6 +11,8 @@ setup: type: ip ipv6: type: ip + ip: + type: ip value: type: long @@ -20,25 +22,25 @@ setup: refresh: true body: - { "index": { } } - - { "ipv4": "192.168.1.10", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f10", "value": 10 } + - { "ipv4": "192.168.1.10", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f10", "value": 10, ip: "192.168.1.10" } - { "index": { } } - - { "ipv4": "192.168.1.12", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f12", "value": 20 } + - { "ipv4": "192.168.1.12", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f12", "value": 20, ip: "2001:db8:a4f8:112a:6001:0:12:7f12" } - { "index": { } } - - { "ipv4": "192.168.1.33", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f33", "value": 40 } + - { "ipv4": "192.168.1.33", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f33", "value": 40, ip: "192.168.1.33" } - { "index": { } } - - { "ipv4": "192.168.1.10", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f10", "value": 20 } + - { "ipv4": "192.168.1.10", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f10", "value": 20, ip: "2001:db8:a4f8:112a:6001:0:12:7f10" } - { "index": { } } - - { "ipv4": "192.168.1.33", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f33", "value": 70 } + - { "ipv4": "192.168.1.33", "ipv6": "2001:db8:a4f8:112a:6001:0:12:7f33", "value": 70, ip: "192.168.1.33" } - { "index": { } } - - { "ipv4": "192.168.2.41", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f41", "value": 20 } + - { "ipv4": "192.168.2.41", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f41", "value": 20, ip: "2001:db8:a4f8:112c:6001:0:12:7f41" } - { "index": { } } - - { "ipv4": "192.168.2.10", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f10", "value": 30 } + - { "ipv4": "192.168.2.10", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f10", "value": 30, ip: "192.168.2.10" } - { "index": { } } - - { "ipv4": "192.168.2.23", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f23", "value": 50 } + - { "ipv4": "192.168.2.23", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f23", "value": 50, ip: "2001:db8:a4f8:112c:6001:0:12:7f23" } - { "index": { } } - - { "ipv4": "192.168.2.41", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f41", "value": 60 } + - { "ipv4": "192.168.2.41", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f41", "value": 60, ip: "192.168.2.41" } - { "index": { } } - - { "ipv4": "192.168.2.10", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f10", "value": 10 } + - { "ipv4": "192.168.2.10", "ipv6": "2001:db8:a4f8:112c:6001:0:12:7f10", "value": 10, ip: "2001:db8:a4f8:112c:6001:0:12:7f10" } --- "IPv4 prefix": @@ -414,3 +416,70 @@ setup: - is_true: aggregations.ip_prefix.buckets.1.is_ipv6 - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } - match: { aggregations.ip_prefix.buckets.1.netmask: null } + +--- +"Mixed IPv4 and IPv6 with is_ipv6 false": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ip" + is_ipv6: false + prefix_len: 16 + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.ip_prefix.buckets: 2 } + - match: { aggregations.ip_prefix.buckets.0.key: "0.18.0.0" } + - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } + - is_false: aggregations.ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.ip_prefix.buckets.0.prefix_len: 16 } + - match: { aggregations.ip_prefix.buckets.0.netmask: "255.255.0.0" } + - match: { aggregations.ip_prefix.buckets.1.key: "192.168.0.0" } + - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } + - is_false: aggregations.ip_prefix.buckets.1.is_ipv6 + - match: { aggregations.ip_prefix.buckets.1.prefix_len: 16 } + - match: { aggregations.ip_prefix.buckets.1.netmask: "255.255.0.0" } + +--- +"Mixed IPv4 and IPv6 with is_ipv6 true": + - skip: + version: " - 8.0.99" + reason: "added in 8.1.0" + - do: + search: + body: + size: 0 + aggs: + ip_prefix: + ip_prefix: + field: "ip" + is_ipv6: true + prefix_len: 64 + + + - match: { hits.total.value: 10 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.ip_prefix.buckets: 3 } + - match: { aggregations.ip_prefix.buckets.0.key: "::" } + - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } + - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 + - match: { aggregations.ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.0.netmask: null } + - match: { aggregations.ip_prefix.buckets.1.key: "2001:db8:a4f8:112a::" } + - match: { aggregations.ip_prefix.buckets.1.doc_count: 2 } + - is_true: aggregations.ip_prefix.buckets.1.is_ipv6 + - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.1.netmask: null } + - match: { aggregations.ip_prefix.buckets.2.key: "2001:db8:a4f8:112c::" } + - match: { aggregations.ip_prefix.buckets.2.doc_count: 3 } + - is_true: aggregations.ip_prefix.buckets.2.is_ipv6 + - match: { aggregations.ip_prefix.buckets.2.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.2.netmask: null } From 318a1393db705d415a627fcdd0e9099a665ab6e8 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Fri, 14 Jan 2022 15:20:19 +0100 Subject: [PATCH 17/36] docs: improve the documentation --- .../bucket/ipprefix-aggregation.asciidoc | 76 ++++++++++++++++--- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc index 6988ae5a5f3dd..1375906810a91 100644 --- a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc @@ -3,8 +3,11 @@ ++++ IP prefix ++++ -IP addresses can be aggregated according to their network or sub-network. IP addresses consists of two groups of bits: the most significant bits which represent the network prefix, and the least significant bits which represent the host. -The network or sub-network is usually identified using a netmask, or a prefix length. The prefix length defines the number of bits representing the network prefix. + +A bucket aggregation that groups documents based on the network or sub-network of an IP address. An IP address consists of two groups of bits: the most significant bits which represent the network prefix, and the least significant bits which represent the host. + +[[ipprefix-agg-ex]] +==== Example For example consider the following index: [source,console] @@ -41,7 +44,7 @@ POST /network-traffic/_bulk?refresh ---------------------------------------------- // TESTSETUP -The following aggregation will aggregate documents in buckets, each identifying a different sub-network. The sub-network is calculated applying a netmask with prefix length of `24` to each IP address in the `ipv4` field: +The following aggregation groups documents into buckets. Each bucket identifies a different sub-network. The sub-network is calculated applying a netmask with prefix length of `24` to each IP address in the `ipv4` field: [source,console,id=ip-prefix-ipv4-example] -------------------------------------------------- @@ -98,7 +101,7 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] -In a similar way we can aggregate IPv6 addresses, making sure we set the `is_ipv6` flag to `true` (see section below): +To aggregate IPv6 addresses, set `is_ipv6` to `true`. [source,console,id=ip-prefix-ipv6-example] -------------------------------------------------- @@ -153,11 +156,12 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] -NOTE: the `netmask` field is not returned in the response when flag `is_ipv6` is set to `true`. +The `netmask` field is not returned in the response when flag `is_ipv6` is set to `true`. +[[ipprefix-agg-keyed-response]] ==== Keyed Response -Setting the `keyed` flag to `true` will associate a unique IP address key with each bucket and return sub-networks as a hash rather than an array: +Set the `keyed` flag of `true` to associate an unique IP address key with each bucket and return sub-networks as a hash rather than an array: Example: @@ -214,9 +218,10 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] -==== Appending the prefix length to the IP address key +[[ipprefix-agg-append-prefix-length]] +==== Append the prefix length to the IP address key -Setting the `append_prefix_len` flag to `true` will return IP address keys catenated with the prefix length of the sub-network: +Set the `append_prefix_len` flag to `true` to catenate IP address keys with the prefix length of the sub-network: Example: @@ -276,9 +281,10 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] +[[ipprefix-agg-min-doc-count]] ==== Minimum document count -It is possible to change the response in such a way that only buckets including a minimum number of documents are returned, using the `min_doc_count` parameter. +Use the `min_doc_count` parameter to only return buckets with a minimum number of documents. [source,console,id=ip-prefix-min-doc-count-example] -------------------------------------------------- @@ -329,6 +335,54 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] -==== IPv6 vs IPv4 addresses and sub-networks +[role="child_attributes"] +[[ip-prefix-agg-params]] +==== Parameters + +`field`:: +(Required, string) +The document IP address field to aggregate on. + +`prefix_len`:: +(Required, integer) +The length of the network prefix. For IPv4 addresses the accepted range is `[0, 32]`. For Ipv6 addresses the accepted range is `[0, 128]`. + +`is_ipv6`:: +(Optional, boolean) +Defines whether the prefix applies to IPv6 addresses. Just specifying the `prefix_len` parameter is not enough to know if an IP prefix applies to IPv4 or IPv6 addresses. Defaults to `false`. + +`append_prefix_len`:: +(Optional, boolean) +Defines whether the prefix length is appended to IP address keys in the response. Defaults to `false`. + +`keyed`:: +(Optional, boolean) +Defines whether buckets are returned as a hash rather than an array in the response. Defaults to `false`. + +`min_doc_count`:: +(Optional, integer) +Defines the minimum number of documents for buckets to be included in the response. Defaults to `1`. + + +[[ipprefix-agg-response]] +==== Response body + +`key`:: +(string) +The IPv6 or IPv4 subnet. + +`prefix_len`:: +(integer) +The length of the prefix used to aggregate the bucket. + +`doc_count`:: +(integer) +Number of documents matching a specific IP prefix. + +`is_ipv6`:: +(boolean) +Defines whether the netmask is an IPv6 netmask. -Just specifying the `prefix_len` parameter is not enough to know if an IP prefix aggregation is done on IPv4 or IPv6 addresses. As a result, the `is_ipv6` flag is needed and defaults to `false`. Failing to set it appropriately will result in unpredictable results due to the way the netmask is applied. +`netmask`:: +(string) +The IPv4 netmask. If `is_ipv6` is `true` in the request this field is missing. From 1aa223f0ec6a6decdf291a8f43e22845288d882e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Fri, 14 Jan 2022 15:22:02 +0100 Subject: [PATCH 18/36] fix: use 1 as the min_doc_count minimum and default value --- .../prefix/IpPrefixAggregationBuilder.java | 5 ++- .../prefix/IpPrefixAggregatorTests.java | 32 +++++++++---------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java index 78da252f74195..d7944907ee80e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java @@ -79,7 +79,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { IpPrefixAggregatorFactory.registerAggregators(builder); } - private long minDocCount = 0; + private long minDocCount = 1; private int prefixLength = -1; private boolean isIpv6 = false; private boolean appendPrefixLength = false; @@ -87,6 +87,9 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { /** Set the minDocCount on this builder, and return the builder so that calls can be chained. */ public IpPrefixAggregationBuilder minDocCount(long minDocCount) { + if (minDocCount < 1) { + throw new IllegalArgumentException("[min_doc_count] must not be less than 1: [" + name + "]"); + } this.minDocCount = minDocCount; return this; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java index 15628bb3e06b9..6296a9dd1dcf8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java @@ -134,7 +134,7 @@ public void testEmptyDocument() throws IOException { .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); final List ipAddresses = Collections.emptyList(); @@ -171,7 +171,7 @@ public void testIpv4Addresses() throws IOException { .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); final List ipAddresses = List.of( @@ -222,7 +222,7 @@ public void testIpv6Addresses() throws IOException { .isIpv6(true) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); final List ipAddresses = List.of( @@ -270,7 +270,7 @@ public void testZeroPrefixLength() throws IOException { .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); final List ipAddresses = List.of( @@ -321,7 +321,7 @@ public void testIpv4MaxPrefixLength() throws IOException { .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); final List ipAddresses = List.of( @@ -372,7 +372,7 @@ public void testIpv6MaxPrefixLength() throws IOException { .isIpv6(true) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); final List ipAddresses = List.of( @@ -421,7 +421,7 @@ public void testAggregateOnIpv4Field() throws IOException { .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType[] fieldTypes = { new IpFieldMapper.IpFieldType(ipv4FieldName), new IpFieldMapper.IpFieldType(ipv6FieldName) }; final List ipAddresses = List.of( @@ -477,7 +477,7 @@ public void testAggregateOnIpv6Field() throws IOException { .isIpv6(true) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType[] fieldTypes = { new IpFieldMapper.IpFieldType(ipv4FieldName), new IpFieldMapper.IpFieldType(ipv6FieldName) }; final List ipAddresses = List.of( @@ -537,7 +537,7 @@ public void testIpv4AggregationAsSubAggregation() throws IOException { .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength) ); final DateFieldMapper.DateFieldType dateFieldType = new DateFieldMapper.DateFieldType(datetimeFieldName); @@ -627,7 +627,7 @@ public void testIpv6AggregationAsSubAggregation() throws IOException { .isIpv6(true) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength) ); final DateFieldMapper.DateFieldType dateFieldType = new DateFieldMapper.DateFieldType(datetimeFieldName); @@ -709,14 +709,14 @@ public void testIpPrefixSubAggregations() throws IOException { .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(topPrefixLength) .subAggregation( new IpPrefixAggregationBuilder(subIpPrefixAggregation).field(ipv4FieldName) .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(subPrefixLength) ); final IpFieldMapper.IpFieldType ipFieldType = new IpFieldMapper.IpFieldType(ipv4FieldName); @@ -790,7 +790,7 @@ public void testIpv4AppendPrefixLength() throws IOException { .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(true) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); final List ipAddresses = List.of( @@ -843,7 +843,7 @@ public void testIpv6AppendPrefixLength() throws IOException { .isIpv6(true) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); final List ipAddresses = List.of( @@ -942,7 +942,7 @@ public void testAggregationWithQueryFilter() throws IOException { .isIpv6(false) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength); final MappedFieldType fieldType = new IpFieldMapper.IpFieldType(field); final List ipAddresses = List.of( @@ -1005,7 +1005,7 @@ public void testMetricAggregation() throws IOException { .isIpv6(true) .keyed(randomBoolean()) .appendPrefixLength(false) - .minDocCount(0) + .minDocCount(1) .prefixLength(prefixLength) .subAggregation(new SumAggregationBuilder(subAggregationName).field(timeField)); final MappedFieldType[] fieldTypes = { From 986f0d870dcbfde5fb0a8d62c2fdd46735959f32 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Fri, 14 Jan 2022 15:28:41 +0100 Subject: [PATCH 19/36] docs: include the field type --- .../reference/aggregations/bucket/ipprefix-aggregation.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc index 1375906810a91..191c1bddb15df 100644 --- a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc @@ -341,7 +341,7 @@ Response: `field`:: (Required, string) -The document IP address field to aggregate on. +The document IP address field to aggregate on. The field mapping type must be `IP`. `prefix_len`:: (Required, integer) From 322e42f412c779e8c0880644b5e10ff70052a7e2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Fri, 14 Jan 2022 15:36:05 +0100 Subject: [PATCH 20/36] docs: rephrase netmask description --- .../reference/aggregations/bucket/ipprefix-aggregation.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc index 191c1bddb15df..52f1711b3b495 100644 --- a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc @@ -385,4 +385,4 @@ Defines whether the netmask is an IPv6 netmask. `netmask`:: (string) -The IPv4 netmask. If `is_ipv6` is `true` in the request this field is missing. +The IPv4 netmask. If `is_ipv6` is `true` in the request, this field is missing in the response. From 63591213d07e43099c2e8593f6c72f1abddd2a3d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Mon, 17 Jan 2022 10:34:39 +0100 Subject: [PATCH 21/36] docs: reference documentation improvements --- .../bucket/ipprefix-aggregation.asciidoc | 113 +++++++++--------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc index 52f1711b3b495..9ff612f0a8f01 100644 --- a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc @@ -9,7 +9,7 @@ A bucket aggregation that groups documents based on the network or sub-network o [[ipprefix-agg-ex]] ==== Example -For example consider the following index: +For example, consider the following index: [source,console] ---------------------------------------------- PUT network-traffic @@ -44,7 +44,7 @@ POST /network-traffic/_bulk?refresh ---------------------------------------------- // TESTSETUP -The following aggregation groups documents into buckets. Each bucket identifies a different sub-network. The sub-network is calculated applying a netmask with prefix length of `24` to each IP address in the `ipv4` field: +The following aggregation groups documents into buckets. Each bucket identifies a different sub-network. The sub-network is calculated by applying a netmask with prefix length of `24` to each IP address in the `ipv4` field: [source,console,id=ip-prefix-ipv4-example] -------------------------------------------------- @@ -121,7 +121,7 @@ GET /network-traffic/_search -------------------------------------------------- // TEST -Response: +If `is_ipv6` is `true`, the response doesn't include a `netmask` for each bucket. [source,console-result] -------------------------------------------------- @@ -156,12 +156,62 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] -The `netmask` field is not returned in the response when flag `is_ipv6` is set to `true`. +[role="child_attributes"] +[[ip-prefix-agg-params]] +==== Parameters + +`field`:: +(Required, string) +The document IP address field to aggregate on. The field mapping type must be <>. + +`prefix_len`:: +(Required, integer) +Length of the network prefix. For IPv4 addresses, the accepted range is `[0, 32]`. For IPv6 addresses, the accepted range is `[0, 128]`. + +`is_ipv6`:: +(Optional, boolean) +Defines whether the prefix applies to IPv6 addresses. Just specifying the `prefix_len` parameter is not enough to know if an IP prefix applies to IPv4 or IPv6 addresses. Defaults to `false`. + +`append_prefix_len`:: +(Optional, boolean) +Defines whether the prefix length is appended to IP address keys in the response. Defaults to `false`. + +`keyed`:: +(Optional, boolean) +Defines whether buckets are returned as a hash rather than an array in the response. Defaults to `false`. + +`min_doc_count`:: +(Optional, integer) +Defines the minimum number of documents for buckets to be included in the response. Defaults to `1`. + + +[[ipprefix-agg-response]] +==== Response body + +`key`:: +(string) +The IPv6 or IPv4 subnet. + +`prefix_len`:: +(integer) +The length of the prefix used to aggregate the bucket. + +`doc_count`:: +(integer) +Number of documents matching a specific IP prefix. + +`is_ipv6`:: +(boolean) +Defines whether the netmask is an IPv6 netmask. + +`netmask`:: +(string) +The IPv4 netmask. If `is_ipv6` is `true` in the request, this field is missing in the response. [[ipprefix-agg-keyed-response]] ==== Keyed Response -Set the `keyed` flag of `true` to associate an unique IP address key with each bucket and return sub-networks as a hash rather than an array: +Set the `keyed` flag of `true` to associate an unique IP address key with each bucket and return sub-networks as a hash rather than an array. Example: @@ -221,7 +271,7 @@ Response: [[ipprefix-agg-append-prefix-length]] ==== Append the prefix length to the IP address key -Set the `append_prefix_len` flag to `true` to catenate IP address keys with the prefix length of the sub-network: +Set the `append_prefix_len` flag to `true` to catenate IP address keys with the prefix length of the sub-network. Example: @@ -335,54 +385,3 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] -[role="child_attributes"] -[[ip-prefix-agg-params]] -==== Parameters - -`field`:: -(Required, string) -The document IP address field to aggregate on. The field mapping type must be `IP`. - -`prefix_len`:: -(Required, integer) -The length of the network prefix. For IPv4 addresses the accepted range is `[0, 32]`. For Ipv6 addresses the accepted range is `[0, 128]`. - -`is_ipv6`:: -(Optional, boolean) -Defines whether the prefix applies to IPv6 addresses. Just specifying the `prefix_len` parameter is not enough to know if an IP prefix applies to IPv4 or IPv6 addresses. Defaults to `false`. - -`append_prefix_len`:: -(Optional, boolean) -Defines whether the prefix length is appended to IP address keys in the response. Defaults to `false`. - -`keyed`:: -(Optional, boolean) -Defines whether buckets are returned as a hash rather than an array in the response. Defaults to `false`. - -`min_doc_count`:: -(Optional, integer) -Defines the minimum number of documents for buckets to be included in the response. Defaults to `1`. - - -[[ipprefix-agg-response]] -==== Response body - -`key`:: -(string) -The IPv6 or IPv4 subnet. - -`prefix_len`:: -(integer) -The length of the prefix used to aggregate the bucket. - -`doc_count`:: -(integer) -Number of documents matching a specific IP prefix. - -`is_ipv6`:: -(boolean) -Defines whether the netmask is an IPv6 netmask. - -`netmask`:: -(string) -The IPv4 netmask. If `is_ipv6` is `true` in the request, this field is missing in the response. From 917389feedbf2f6b58892e4b7fbd76c78f6eb092 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 10:25:54 +0100 Subject: [PATCH 22/36] fix: use prefix_length instead of prefix_len --- .../bucket/ipprefix-aggregation.asciidoc | 50 +++++----- .../test/search.aggregation/450_ip_prefix.yml | 92 +++++++++---------- .../prefix/IpPrefixAggregationBuilder.java | 4 +- 3 files changed, 73 insertions(+), 73 deletions(-) diff --git a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc index 9ff612f0a8f01..2dee6654869f7 100644 --- a/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/ipprefix-aggregation.asciidoc @@ -55,7 +55,7 @@ GET /network-traffic/_search "ipv4-subnets": { "ip_prefix": { "field": "ipv4", - "prefix_len": 24 + "prefix_length": 24 } } } @@ -77,21 +77,21 @@ Response: "key": "192.168.1.0", "is_ipv6": false, "doc_count": 4, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" }, { "key": "192.168.2.0", "is_ipv6": false, "doc_count": 3, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" }, { "key": "192.168.3.0", "is_ipv6": false, "doc_count": 2, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" } ] @@ -112,7 +112,7 @@ GET /network-traffic/_search "ipv6-subnets": { "ip_prefix": { "field": "ipv6", - "prefix_len": 64, + "prefix_length": 64, "is_ipv6": true } } @@ -135,19 +135,19 @@ If `is_ipv6` is `true`, the response doesn't include a `netmask` for each bucket "key": "2001:db8:a4f8:112a::", "is_ipv6": true, "doc_count": 4, - "prefix_len": 64 + "prefix_length": 64 }, { "key": "2001:db8:a4f8:112c::", "is_ipv6": true, "doc_count": 3, - "prefix_len": 64 + "prefix_length": 64 }, { "key": "2001:db8:a4f8:114f::", "is_ipv6": true, "doc_count": 2, - "prefix_len": 64 + "prefix_length": 64 } ] } @@ -164,15 +164,15 @@ If `is_ipv6` is `true`, the response doesn't include a `netmask` for each bucket (Required, string) The document IP address field to aggregate on. The field mapping type must be <>. -`prefix_len`:: +`prefix_length`:: (Required, integer) Length of the network prefix. For IPv4 addresses, the accepted range is `[0, 32]`. For IPv6 addresses, the accepted range is `[0, 128]`. `is_ipv6`:: (Optional, boolean) -Defines whether the prefix applies to IPv6 addresses. Just specifying the `prefix_len` parameter is not enough to know if an IP prefix applies to IPv4 or IPv6 addresses. Defaults to `false`. +Defines whether the prefix applies to IPv6 addresses. Just specifying the `prefix_length` parameter is not enough to know if an IP prefix applies to IPv4 or IPv6 addresses. Defaults to `false`. -`append_prefix_len`:: +`append_prefix_length`:: (Optional, boolean) Defines whether the prefix length is appended to IP address keys in the response. Defaults to `false`. @@ -192,7 +192,7 @@ Defines the minimum number of documents for buckets to be included in the respon (string) The IPv6 or IPv4 subnet. -`prefix_len`:: +`prefix_length`:: (integer) The length of the prefix used to aggregate the bucket. @@ -224,7 +224,7 @@ GET /network-traffic/_search "ipv4-subnets": { "ip_prefix": { "field": "ipv4", - "prefix_len": 24, + "prefix_length": 24, "keyed": true } } @@ -246,19 +246,19 @@ Response: "192.168.1.0": { "is_ipv6": false, "doc_count": 4, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" }, "192.168.2.0": { "is_ipv6": false, "doc_count": 3, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" }, "192.168.3.0": { "is_ipv6": false, "doc_count": 2, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" } } @@ -271,7 +271,7 @@ Response: [[ipprefix-agg-append-prefix-length]] ==== Append the prefix length to the IP address key -Set the `append_prefix_len` flag to `true` to catenate IP address keys with the prefix length of the sub-network. +Set the `append_prefix_length` flag to `true` to catenate IP address keys with the prefix length of the sub-network. Example: @@ -284,8 +284,8 @@ GET /network-traffic/_search "ipv4-subnets": { "ip_prefix": { "field": "ipv4", - "prefix_len": 24, - "append_prefix_len": true + "prefix_length": 24, + "append_prefix_length": true } } } @@ -307,21 +307,21 @@ Response: "key": "192.168.1.0/24", "is_ipv6": false, "doc_count": 4, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" }, { "key": "192.168.2.0/24", "is_ipv6": false, "doc_count": 3, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" }, { "key": "192.168.3.0/24", "is_ipv6": false, "doc_count": 2, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" } ] @@ -345,7 +345,7 @@ GET /network-traffic/_search "ipv4-subnets": { "ip_prefix": { "field": "ipv4", - "prefix_len": 24, + "prefix_length": 24, "min_doc_count": 3 } } @@ -368,14 +368,14 @@ Response: "key": "192.168.1.0", "is_ipv6": false, "doc_count": 4, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" }, { "key": "192.168.2.0", "is_ipv6": false, "doc_count": 3, - "prefix_len": 24, + "prefix_length": 24, "netmask": "255.255.255.0" } ] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml index 8d63be257577b..0d99b2ae1ce87 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml @@ -56,7 +56,7 @@ setup: ip_prefix: field: "ipv4" is_ipv6: false - prefix_len: 24 + prefix_length: 24 - match: { hits.total.value: 10 } @@ -65,16 +65,16 @@ setup: - match: { aggregations.ip_prefix.buckets.0.key: "192.168.1.0" } - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } - is_false: aggregations.ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.ip_prefix.buckets.0.prefix_len: 24 } + - match: { aggregations.ip_prefix.buckets.0.prefix_length: 24 } - match: { aggregations.ip_prefix.buckets.0.netmask: "255.255.255.0" } - match: { aggregations.ip_prefix.buckets.1.key: "192.168.2.0" } - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } - is_false: aggregations.ip_prefix.buckets.1.is_ipv6 - - match: { aggregations.ip_prefix.buckets.1.prefix_len: 24 } + - match: { aggregations.ip_prefix.buckets.1.prefix_length: 24 } - match: { aggregations.ip_prefix.buckets.1.netmask: "255.255.255.0" } --- -# NOTE: here prefix_len = 24 which means the netmask 255.255.255.0 will be applied to the +# NOTE: here prefix_length = 24 which means the netmask 255.255.255.0 will be applied to the # high 24 bits of a field which is an IPv4 address encoded on 16 bytes. As a result the # network part will just 0s. "IPv4 prefix with incorrect is_ipv6": @@ -90,7 +90,7 @@ setup: ip_prefix: field: "ipv4" is_ipv6: true - prefix_len: 24 + prefix_length: 24 - match: { hits.total.value: 10 } @@ -99,7 +99,7 @@ setup: - match: { aggregations.ip_prefix.buckets.0.key: "::" } - match: { aggregations.ip_prefix.buckets.0.doc_count: 10 } - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.ip_prefix.buckets.0.prefix_len: 24 } + - match: { aggregations.ip_prefix.buckets.0.prefix_length: 24 } --- "IPv4 short prefix": @@ -115,12 +115,12 @@ setup: ip_prefix: field: "ipv4" is_ipv6: false - prefix_len: 13 + prefix_length: 13 second: ip_prefix: field: "ipv4" is_ipv6: false - prefix_len: 6 + prefix_length: 6 - match: { hits.total.value: 10 } @@ -129,13 +129,13 @@ setup: - match: { aggregations.first.buckets.0.key: "192.168.0.0" } - match: { aggregations.first.buckets.0.doc_count: 10 } - is_false: aggregations.first.buckets.0.is_ipv6 - - match: { aggregations.first.buckets.0.prefix_len: 13 } + - match: { aggregations.first.buckets.0.prefix_length: 13 } - match: { aggregations.first.buckets.0.netmask: "255.248.0.0" } - length: { aggregations.second.buckets: 1 } - match: { aggregations.second.buckets.0.key: "192.0.0.0" } - match: { aggregations.second.buckets.0.doc_count: 10 } - is_false: aggregations.second.buckets.0.is_ipv6 - - match: { aggregations.second.buckets.0.prefix_len: 6 } + - match: { aggregations.second.buckets.0.prefix_length: 6 } - match: { aggregations.second.buckets.0.netmask: "252.0.0.0" } --- @@ -152,7 +152,7 @@ setup: ip_prefix: field: "ipv6" is_ipv6: true - prefix_len: 64 + prefix_length: 64 - match: { hits.total.value: 10 } - match: { hits.total.relation: "eq" } @@ -160,16 +160,16 @@ setup: - match: { aggregations.ip_prefix.buckets.0.key: "2001:db8:a4f8:112a::" } - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.0.prefix_length: 64 } - match: { aggregations.ip_prefix.buckets.0.netmask: null } - match: { aggregations.ip_prefix.buckets.1.key: "2001:db8:a4f8:112c::" } - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } - is_true: aggregations.ip_prefix.buckets.1.is_ipv6 - - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.1.prefix_length: 64 } - match: { aggregations.ip_prefix.buckets.1.netmask: null } --- -# NOTE: here prefix_len = 16 which means the netmask will be applied to the second +# NOTE: here prefix_length = 16 which means the netmask will be applied to the second # group of 2 bytes starting from the right (i.e. for "2001:db8:a4f8:112a:6001:0:12:7f10" # it will be the 2 bytes whose value is set to 12 hexadecimal) which results to 18 decimal, # with everything else being 0s. @@ -186,7 +186,7 @@ setup: ip_prefix: field: "ipv6" is_ipv6: false - prefix_len: 16 + prefix_length: 16 - match: { hits.total.value: 10 } @@ -195,7 +195,7 @@ setup: - match: { aggregations.ip_prefix.buckets.0.key: "0.18.0.0" } - match: { aggregations.ip_prefix.buckets.0.doc_count: 10 } - is_false: aggregations.ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.ip_prefix.buckets.0.prefix_len: 16 } + - match: { aggregations.ip_prefix.buckets.0.prefix_length: 16 } - match: { aggregations.ip_prefix.buckets.0.netmask: "255.255.0.0" } --- @@ -213,7 +213,7 @@ setup: ip_prefix: field: "ipv4" is_ipv6: false - prefix_len: 44 + prefix_length: 44 --- @@ -231,7 +231,7 @@ setup: ip_prefix: field: "ipv6" is_ipv6: true - prefix_len: 170 + prefix_length: 170 --- "IPv4 prefix sub aggregation": @@ -247,13 +247,13 @@ setup: ip_prefix: field: "ipv4" is_ipv6: false - prefix_len: 16 + prefix_length: 16 aggs: sub_ip_prefix: ip_prefix: field: "ipv4" is_ipv6: false - prefix_len: 24 + prefix_length: 24 - match: { hits.total.value: 10 } @@ -262,17 +262,17 @@ setup: - match: { aggregations.top_ip_prefix.buckets.0.key: "192.168.0.0" } - match: { aggregations.top_ip_prefix.buckets.0.doc_count: 10 } - is_false: aggregations.top_ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.top_ip_prefix.buckets.0.prefix_len: 16 } + - match: { aggregations.top_ip_prefix.buckets.0.prefix_length: 16 } - match: { aggregations.top_ip_prefix.buckets.0.netmask: "255.255.0.0" } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.key: "192.168.1.0" } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.doc_count: 5 } - is_false: aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.prefix_len: 24 } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.prefix_length: 24 } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.netmask: "255.255.255.0" } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.key: "192.168.2.0" } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.doc_count: 5 } - is_false: aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.is_ipv6 - - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.prefix_len: 24 } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.prefix_length: 24 } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.netmask: "255.255.255.0" } --- @@ -289,13 +289,13 @@ setup: ip_prefix: field: "ipv6" is_ipv6: true - prefix_len: 48 + prefix_length: 48 aggs: sub_ip_prefix: ip_prefix: field: "ipv6" is_ipv6: true - prefix_len: 64 + prefix_length: 64 - match: { hits.total.value: 10 } @@ -304,16 +304,16 @@ setup: - match: { aggregations.top_ip_prefix.buckets.0.key: "2001:db8:a4f8::" } - match: { aggregations.top_ip_prefix.buckets.0.doc_count: 10 } - is_true: aggregations.top_ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.top_ip_prefix.buckets.0.prefix_len: 48 } + - match: { aggregations.top_ip_prefix.buckets.0.prefix_length: 48 } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.key: "2001:db8:a4f8:112a::" } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.doc_count: 5 } - is_true: aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.prefix_length: 64 } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.0.netmask: null } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.key: "2001:db8:a4f8:112c::" } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.doc_count: 5 } - is_true: aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.is_ipv6 - - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.prefix_length: 64 } - match: { aggregations.top_ip_prefix.buckets.0.sub_ip_prefix.buckets.1.netmask: null } --- @@ -330,7 +330,7 @@ setup: ip_prefix: field: "ipv6" is_ipv6: true - prefix_len: 64 + prefix_length: 64 aggs: sum: sum: @@ -343,13 +343,13 @@ setup: - match: { aggregations.ip_prefix.buckets.0.key: "2001:db8:a4f8:112a::" } - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.0.prefix_length: 64 } - match: { aggregations.ip_prefix.buckets.0.netmask: null } - match: { aggregations.ip_prefix.buckets.0.sum.value: 160 } - match: { aggregations.ip_prefix.buckets.1.key: "2001:db8:a4f8:112c::" } - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } - is_true: aggregations.ip_prefix.buckets.1.is_ipv6 - - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.1.prefix_length: 64 } - match: { aggregations.ip_prefix.buckets.1.netmask: null } - match: { aggregations.ip_prefix.buckets.1.sum.value: 170 } @@ -367,8 +367,8 @@ setup: ip_prefix: field: "ipv4" is_ipv6: false - prefix_len: 24 - append_prefix_len: true + prefix_length: 24 + append_prefix_length: true - match: { hits.total.value: 10 } @@ -377,12 +377,12 @@ setup: - match: { aggregations.ip_prefix.buckets.0.key: "192.168.1.0/24" } - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } - is_false: aggregations.ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.ip_prefix.buckets.0.prefix_len: 24 } + - match: { aggregations.ip_prefix.buckets.0.prefix_length: 24 } - match: { aggregations.ip_prefix.buckets.0.netmask: "255.255.255.0" } - match: { aggregations.ip_prefix.buckets.1.key: "192.168.2.0/24" } - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } - is_false: aggregations.ip_prefix.buckets.1.is_ipv6 - - match: { aggregations.ip_prefix.buckets.1.prefix_len: 24 } + - match: { aggregations.ip_prefix.buckets.1.prefix_length: 24 } - match: { aggregations.ip_prefix.buckets.1.netmask: "255.255.255.0" } --- @@ -399,8 +399,8 @@ setup: ip_prefix: field: "ipv6" is_ipv6: true - prefix_len: 64 - append_prefix_len: true + prefix_length: 64 + append_prefix_length: true - match: { hits.total.value: 10 } @@ -409,12 +409,12 @@ setup: - match: { aggregations.ip_prefix.buckets.0.key: "2001:db8:a4f8:112a::/64" } - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.0.prefix_length: 64 } - match: { aggregations.ip_prefix.buckets.0.netmask: null } - match: { aggregations.ip_prefix.buckets.1.key: "2001:db8:a4f8:112c::/64" } - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } - is_true: aggregations.ip_prefix.buckets.1.is_ipv6 - - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.1.prefix_length: 64 } - match: { aggregations.ip_prefix.buckets.1.netmask: null } --- @@ -431,7 +431,7 @@ setup: ip_prefix: field: "ip" is_ipv6: false - prefix_len: 16 + prefix_length: 16 - match: { hits.total.value: 10 } @@ -440,12 +440,12 @@ setup: - match: { aggregations.ip_prefix.buckets.0.key: "0.18.0.0" } - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } - is_false: aggregations.ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.ip_prefix.buckets.0.prefix_len: 16 } + - match: { aggregations.ip_prefix.buckets.0.prefix_length: 16 } - match: { aggregations.ip_prefix.buckets.0.netmask: "255.255.0.0" } - match: { aggregations.ip_prefix.buckets.1.key: "192.168.0.0" } - match: { aggregations.ip_prefix.buckets.1.doc_count: 5 } - is_false: aggregations.ip_prefix.buckets.1.is_ipv6 - - match: { aggregations.ip_prefix.buckets.1.prefix_len: 16 } + - match: { aggregations.ip_prefix.buckets.1.prefix_length: 16 } - match: { aggregations.ip_prefix.buckets.1.netmask: "255.255.0.0" } --- @@ -462,7 +462,7 @@ setup: ip_prefix: field: "ip" is_ipv6: true - prefix_len: 64 + prefix_length: 64 - match: { hits.total.value: 10 } @@ -471,15 +471,15 @@ setup: - match: { aggregations.ip_prefix.buckets.0.key: "::" } - match: { aggregations.ip_prefix.buckets.0.doc_count: 5 } - is_true: aggregations.ip_prefix.buckets.0.is_ipv6 - - match: { aggregations.ip_prefix.buckets.0.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.0.prefix_length: 64 } - match: { aggregations.ip_prefix.buckets.0.netmask: null } - match: { aggregations.ip_prefix.buckets.1.key: "2001:db8:a4f8:112a::" } - match: { aggregations.ip_prefix.buckets.1.doc_count: 2 } - is_true: aggregations.ip_prefix.buckets.1.is_ipv6 - - match: { aggregations.ip_prefix.buckets.1.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.1.prefix_length: 64 } - match: { aggregations.ip_prefix.buckets.1.netmask: null } - match: { aggregations.ip_prefix.buckets.2.key: "2001:db8:a4f8:112c::" } - match: { aggregations.ip_prefix.buckets.2.doc_count: 3 } - is_true: aggregations.ip_prefix.buckets.2.is_ipv6 - - match: { aggregations.ip_prefix.buckets.2.prefix_len: 64 } + - match: { aggregations.ip_prefix.buckets.2.prefix_length: 64 } - match: { aggregations.ip_prefix.buckets.2.netmask: null } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java index d7944907ee80e..27db95e736315 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java @@ -46,9 +46,9 @@ public class IpPrefixAggregationBuilder extends ValuesSourceAggregationBuilder Date: Thu, 20 Jan 2022 11:32:24 +0100 Subject: [PATCH 23/36] fix: improve exception error messages --- .../test/search.aggregation/450_ip_prefix.yml | 4 +- .../prefix/IpPrefixAggregationBuilder.java | 78 ++++++++++--------- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml index 0d99b2ae1ce87..0c1d09b2e770f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/450_ip_prefix.yml @@ -204,7 +204,7 @@ setup: version: " - 8.0.99" reason: "added in 8.1.0" - do: - catch: /\[prefix_len\] must be in range \[0, 32\] for aggregation \[ip_prefix\]/ + catch: /\[prefix_length\] must be in range \[0, 32\] while value is \[44\]/ search: body: size: 0 @@ -222,7 +222,7 @@ setup: version: " - 8.0.99" reason: "added in 8.1.0" - do: - catch: /\[prefix_len\] must be in range \[0, 128\] for aggregation \[ip_prefix\]/ + catch: /\[prefix_length] must be in range \[0, 128\] while value is \[170]/ search: body: size: 0 diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java index 27db95e736315..d2f460d60ae6c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java @@ -85,10 +85,33 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { private boolean appendPrefixLength = false; private boolean keyed = false; + private static void throwOnInvalidFieldValue( + final String fieldName, + final T minValue, + final T maxValue, + final T fieldValue + ) { + throw new IllegalArgumentException( + "[" + + fieldName + + "] must be in range [" + + minValue.toString() + + ", " + + maxValue.toString() + + "] while value is [" + + fieldValue.toString() + "]" + ); + } + /** Set the minDocCount on this builder, and return the builder so that calls can be chained. */ public IpPrefixAggregationBuilder minDocCount(long minDocCount) { if (minDocCount < 1) { - throw new IllegalArgumentException("[min_doc_count] must not be less than 1: [" + name + "]"); + throwOnInvalidFieldValue( + MIN_DOC_COUNT_FIELD.getPreferredName(), + 1, + Integer.MAX_VALUE, + minDocCount + ); } this.minDocCount = minDocCount; return this; @@ -101,7 +124,12 @@ public IpPrefixAggregationBuilder minDocCount(long minDocCount) { * */ public IpPrefixAggregationBuilder prefixLength(int prefixLength) { if (prefixLength < MIN_PREFIX_LENGTH) { - throw new IllegalArgumentException("[prefix_len] must not be less than " + MIN_PREFIX_LENGTH + ": [" + name + "]"); + throwOnInvalidFieldValue( + PREFIX_LENGTH_FIELD.getPreferredName(), + 0, + isIpv6 ? IPV6_MAX_PREFIX_LENGTH : IPV4_MAX_PREFIX_LENGTH, + prefixLength + ); } this.prefixLength = prefixLength; return this; @@ -186,31 +214,12 @@ protected ValuesSourceAggregatorFactory innerBuild( ) throws IOException { IpPrefixAggregationSupplier aggregationSupplier = context.getValuesSourceRegistry().getAggregator(REGISTRY_KEY, config); - if (isIpv6 && prefixLength > IPV6_MAX_PREFIX_LENGTH) { - throw new IllegalArgumentException( - "[" - + PREFIX_LENGTH_FIELD.getPreferredName() - + "] must be in range [" - + MIN_PREFIX_LENGTH - + ", " - + IPV6_MAX_PREFIX_LENGTH - + "] for aggregation [" - + this.getName() - + "]" - ); - } - - if (isIpv6 == false && prefixLength > IPV4_MAX_PREFIX_LENGTH) { - throw new IllegalArgumentException( - "[" - + PREFIX_LENGTH_FIELD.getPreferredName() - + "] must be in range [" - + MIN_PREFIX_LENGTH - + ", " - + IPV4_MAX_PREFIX_LENGTH - + "] for aggregation [" - + this.getName() - + "]" + if (prefixLength < 0 || (isIpv6 == false && prefixLength > IPV4_MAX_PREFIX_LENGTH) || (isIpv6 && prefixLength > IPV6_MAX_PREFIX_LENGTH)) { + throwOnInvalidFieldValue( + PREFIX_LENGTH_FIELD.getPreferredName(), + MIN_PREFIX_LENGTH, + isIpv6 ? IPV6_MAX_PREFIX_LENGTH : IPV4_MAX_PREFIX_LENGTH, + prefixLength ); } @@ -246,15 +255,12 @@ protected ValuesSourceAggregatorFactory innerBuild( * network, or is not in range [0, 32] for an IPv4 network. */ public static BytesRef extractNetmask(int prefixLength, boolean isIpv6) { - if (prefixLength < 0 || (isIpv6 == false && prefixLength > 32) || (isIpv6 && prefixLength > 128)) { - throw new IllegalArgumentException( - "[" - + PREFIX_LENGTH_FIELD.getPreferredName() - + "] must be in range [" - + MIN_PREFIX_LENGTH - + ", " - + (isIpv6 ? IPV6_MAX_PREFIX_LENGTH : IPV4_MAX_PREFIX_LENGTH) - + "]" + if (prefixLength < 0 || (isIpv6 == false && prefixLength > IPV4_MAX_PREFIX_LENGTH) || (isIpv6 && prefixLength > IPV6_MAX_PREFIX_LENGTH)) { + throwOnInvalidFieldValue( + PREFIX_LENGTH_FIELD.getPreferredName(), + MIN_PREFIX_LENGTH, + isIpv6 ? IPV6_MAX_PREFIX_LENGTH : IPV4_MAX_PREFIX_LENGTH, + prefixLength ); } From cdfed61dad1c63dbf9ff0b2c7df114918ef023f5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 11:37:12 +0100 Subject: [PATCH 24/36] fix: add some notes clarifying netmask setting --- .../bucket/prefix/IpPrefixAggregationBuilder.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java index d2f460d60ae6c..f1dc5cbfb140d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java @@ -270,9 +270,14 @@ public static BytesRef extractNetmask(int prefixLength, boolean isIpv6) { int bytesCount = prefixLength / 8; int bitsCount = prefixLength % 8; int i = 0; + //NOTE: first set whole bytes to 255 (0xFF) for (; i < bytesCount; i++) { ipAddress[i] = (byte) 0xFF; } + //NOTE: then set the remaining bits to 1. + //Trailing bits are already set to 0 at initialization time. + //Example: for prefixLength = 20, we first set 16 bits (2 bytes) + //to 0xFF, then set the remaining 4 bits to 1. if (bitsCount > 0) { int rem = 0; for (int j = 0; j < bitsCount; j++) { From 785eb6f5063f965072f386487b8605531a35893a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 11:44:10 +0100 Subject: [PATCH 25/36] fix: use ValueSourceConfig instead of ValueSource and DocValueFormat --- .../prefix/IpPrefixAggregationSupplier.java | 4 ++-- .../bucket/prefix/IpPrefixAggregator.java | 22 ++++++++----------- .../prefix/IpPrefixAggregatorFactory.java | 6 ++--- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java index 2634f1da1ace0..58807902232ac 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; import java.util.Map; @@ -22,8 +23,7 @@ public interface IpPrefixAggregationSupplier { Aggregator build( String name, AggregatorFactories factories, - ValuesSource valuesSource, - DocValueFormat format, + ValuesSourceConfig config, boolean keyed, long minDocCount, IpPrefixAggregator.IpPrefix ipPrefix, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java index 6b77900f5aa5e..17f7bdd46007e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java @@ -13,7 +13,6 @@ import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -26,7 +25,7 @@ import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; import org.elasticsearch.search.aggregations.bucket.terms.BytesKeyedBucketOrds; import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; import java.util.ArrayList; @@ -87,8 +86,7 @@ public int hashCode() { } } - final ValuesSource.Bytes valuesSource; - final DocValueFormat format; + final ValuesSourceConfig config; final long minDocCount; final boolean keyed; final BytesKeyedBucketOrds bucketOrds; @@ -97,8 +95,7 @@ public int hashCode() { public IpPrefixAggregator( String name, AggregatorFactories factories, - ValuesSource valuesSource, - DocValueFormat format, + ValuesSourceConfig config, boolean keyed, long minDocCount, IpPrefix ipPrefix, @@ -108,8 +105,7 @@ public IpPrefixAggregator( Map metadata ) throws IOException { super(name, factories, context, parent, CardinalityUpperBound.MANY, metadata); - this.valuesSource = (ValuesSource.Bytes) valuesSource; - this.format = format; + this.config = config; this.keyed = keyed; this.minDocCount = minDocCount; this.bucketOrds = BytesKeyedBucketOrds.build(bigArrays(), cardinality); @@ -118,9 +114,9 @@ public IpPrefixAggregator( @Override protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { - return valuesSource == null + return config.getValuesSource() == null ? LeafBucketCollector.NO_OP_COLLECTOR - : new IpPrefixLeafCollector(sub, valuesSource.bytesValues(ctx), ipPrefix); + : new IpPrefixLeafCollector(sub, config.getValuesSource().bytesValues(ctx), ipPrefix); } private class IpPrefixLeafCollector extends LeafBucketCollectorBase { @@ -238,7 +234,7 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I long docCount = bucketDocCount(ordinal); buckets.add( new InternalIpPrefix.Bucket( - format, + config.format(), BytesRef.deepCopyOf(ipAddress), keyed, ipPrefix.isIpv6, @@ -252,14 +248,14 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I // NOTE: the aggregator is expected to return sorted results CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator()); } - results[ordIdx] = new InternalIpPrefix(name, format, keyed, minDocCount, buckets, metadata()); + results[ordIdx] = new InternalIpPrefix(name, config.format(), keyed, minDocCount, buckets, metadata()); } return results; } @Override public InternalAggregation buildEmptyAggregation() { - return new InternalIpPrefix(name, format, keyed, minDocCount, Collections.emptyList(), metadata()); + return new InternalIpPrefix(name, config.format(), keyed, minDocCount, Collections.emptyList(), metadata()); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java index 83bef53f8bf2f..90eb30b4f7118 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java @@ -55,8 +55,7 @@ protected Aggregator createUnmapped(Aggregator parent, Map metad return new IpPrefixAggregator( name, factories, - null, - config.format(), + config, keyed, minDocCount, ipPrefix, @@ -73,8 +72,7 @@ protected Aggregator doCreateInternal(Aggregator parent, CardinalityUpperBound c return aggregationSupplier.build( name, factories, - config.getValuesSource(), - config.format(), + config, keyed, minDocCount, ipPrefix, From 54547e444ab906a8c0c8fb60623475f3dce7c3c7 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 11:50:25 +0100 Subject: [PATCH 26/36] fix: remove check of bucket limit --- .../aggregations/bucket/prefix/IpPrefixAggregator.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java index 17f7bdd46007e..fb9245d706b33 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java @@ -195,11 +195,6 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I bucketsInOrd[ordIdx] = (int) bucketCount; totalOrdsToCollect += bucketCount; } - if (totalOrdsToCollect > Integer.MAX_VALUE) { - throw new AggregationExecutionException( - "Can't collect more than [" + Integer.MAX_VALUE + "] buckets but attempted [" + totalOrdsToCollect + "]" - ); - } long[] bucketOrdsToCollect = new long[(int) totalOrdsToCollect]; int b = 0; From 130cb9f625cabe7e5837df95b5ef4f8b8b251bbf Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 12:18:41 +0100 Subject: [PATCH 27/36] fix: use a NonCollectingAggregator when unmapped --- .../bucket/prefix/IpPrefixAggregator.java | 36 +++++++++++++++++++ .../prefix/IpPrefixAggregatorFactory.java | 4 +-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java index fb9245d706b33..a81763e48c310 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java @@ -22,6 +22,7 @@ import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.NonCollectingAggregator; import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; import org.elasticsearch.search.aggregations.bucket.terms.BytesKeyedBucketOrds; import org.elasticsearch.search.aggregations.support.AggregationContext; @@ -257,4 +258,39 @@ public InternalAggregation buildEmptyAggregation() { public void doClose() { Releasables.close(bucketOrds); } + + public static class Unmapped extends NonCollectingAggregator { + + private final ValuesSourceConfig config; + private final boolean keyed; + private final long minDocCount; + + protected Unmapped( + String name, + AggregatorFactories factories, + ValuesSourceConfig config, + boolean keyed, + long minDocCount, + AggregationContext context, + Aggregator parent, + Map metadata + ) throws IOException { + super(name, context, parent, factories, metadata); + this.config = config; + this.keyed = keyed; + this.minDocCount = minDocCount; + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalIpPrefix( + name, + config.format(), + keyed, + minDocCount, + Collections.emptyList(), + metadata() + ); + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java index 90eb30b4f7118..c953cb22cc8e5 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java @@ -52,16 +52,14 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { @Override protected Aggregator createUnmapped(Aggregator parent, Map metadata) throws IOException { - return new IpPrefixAggregator( + return new IpPrefixAggregator.Unmapped( name, factories, config, keyed, minDocCount, - ipPrefix, context, parent, - CardinalityUpperBound.NONE, metadata ); } From ccab36806f5b4622094f3f5f36f75ca643b8f320 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 13:04:05 +0100 Subject: [PATCH 28/36] fix: improve performance of critical path removing Arrays.copyOfRange --- .../bucket/prefix/IpPrefixAggregator.java | 47 +++++++------------ 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java index a81763e48c310..bbadf20fa8b97 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java @@ -137,20 +137,15 @@ public void collect(int doc, long owningBucketOrd) throws IOException { if (values.advanceExact(doc)) { int valuesCount = values.docValueCount(); - byte[] previousSubnet = null; + BytesRef previousSubnet = null; + BytesRef subnet; for (int i = 0; i < valuesCount; ++i) { - BytesRef value = values.nextValue(); - byte[] ipAddress = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); - byte[] netmask = Arrays.copyOfRange( - ipPrefix.netmask.bytes, - ipPrefix.netmask.offset, - ipPrefix.netmask.offset + ipPrefix.netmask.length - ); - byte[] subnet = maskIpAddress(ipAddress, netmask); - if (Arrays.equals(subnet, previousSubnet)) { + BytesRef ipAddress = values.nextValue(); + subnet = maskIpAddress(ipAddress, ipPrefix.netmask); + if (previousSubnet != null && subnet.bytesEquals(previousSubnet)) { continue; } - long bucketOrd = bucketOrds.add(owningBucketOrd, new BytesRef(subnet)); + long bucketOrd = bucketOrds.add(owningBucketOrd, subnet); if (bucketOrd < 0) { bucketOrd = -1 - bucketOrd; collectExistingBucket(sub, doc, bucketOrd); @@ -162,28 +157,22 @@ public void collect(int doc, long owningBucketOrd) throws IOException { } } - private byte[] maskIpAddress(byte[] ipAddress, byte[] netmask) { - // NOTE: ip addresses are always encoded to 16 bytes by IpFieldMapper - if (ipAddress.length != 16) { - throw new IllegalArgumentException("Invalid length for ip address [" + ipAddress.length + "]"); - } - if (netmask.length == 4) { - return mask(Arrays.copyOfRange(ipAddress, 12, 16), netmask); - } - if (netmask.length == 16) { - return mask(ipAddress, netmask); - } - - throw new IllegalArgumentException("Invalid length for netmask [" + netmask.length + "]"); + private BytesRef maskIpAddress(final BytesRef ipAddress, final BytesRef netmask) { + assert ipAddress.length == 16 : "Invalid length for ip address [" + ipAddress.length + "] expected 16 bytes"; + return mask(ipAddress, netmask); } - private byte[] mask(byte[] ipAddress, byte[] subnetMask) { - byte[] subnet = new byte[ipAddress.length]; - for (int i = 0; i < ipAddress.length; ++i) { - subnet[i] = (byte) (ipAddress[i] & subnetMask[i]); + private BytesRef mask(final BytesRef ipAddress, final BytesRef subnetMask) { + //NOTE: IPv4 addresses are encoded as 16-bytes. As a result, we use an + //offset (12) to apply the subnet to the last 4 bytes (byes 12, 13, 14, 15) + //if the subnet mask is just a 4-bytes subnet mask. + int offset = subnetMask.length == 4 ? 12 : 0; + byte[] subnet = new byte[subnetMask.length]; + for (int i = 0; i < subnetMask.length; ++i) { + subnet[i] = (byte) (ipAddress.bytes[i + offset] & subnetMask.bytes[i]); } - return subnet; + return new BytesRef(subnet); } } From df8ff71795ff89ac5c7fe11e1165e6e6a3278ee9 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 14:39:25 +0100 Subject: [PATCH 29/36] fix: avoid re-allocation of BytesRef variables --- .../aggregations/bucket/prefix/IpPrefixAggregator.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java index bbadf20fa8b97..097bef9697320 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java @@ -134,13 +134,14 @@ private class IpPrefixLeafCollector extends LeafBucketCollectorBase { @Override public void collect(int doc, long owningBucketOrd) throws IOException { + BytesRef previousSubnet = null; + BytesRef subnet; + BytesRef ipAddress; if (values.advanceExact(doc)) { int valuesCount = values.docValueCount(); - BytesRef previousSubnet = null; - BytesRef subnet; for (int i = 0; i < valuesCount; ++i) { - BytesRef ipAddress = values.nextValue(); + ipAddress = values.nextValue(); subnet = maskIpAddress(ipAddress, ipPrefix.netmask); if (previousSubnet != null && subnet.bytesEquals(previousSubnet)) { continue; From 6602d96666e3db9ef664aa90357c864c2161bf8a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 14:50:00 +0100 Subject: [PATCH 30/36] fix: code format violations --- .../prefix/IpPrefixAggregationBuilder.java | 35 ++++++++----------- .../prefix/IpPrefixAggregationSupplier.java | 2 -- .../bucket/prefix/IpPrefixAggregator.java | 16 +++------ .../prefix/IpPrefixAggregatorFactory.java | 24 ++----------- 4 files changed, 21 insertions(+), 56 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java index f1dc5cbfb140d..3e4ee317c36fa 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java @@ -85,12 +85,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { private boolean appendPrefixLength = false; private boolean keyed = false; - private static void throwOnInvalidFieldValue( - final String fieldName, - final T minValue, - final T maxValue, - final T fieldValue - ) { + private static void throwOnInvalidFieldValue(final String fieldName, final T minValue, final T maxValue, final T fieldValue) { throw new IllegalArgumentException( "[" + fieldName @@ -99,19 +94,15 @@ private static void throwOnInvalidFieldValue( + ", " + maxValue.toString() + "] while value is [" - + fieldValue.toString() + "]" + + fieldValue.toString() + + "]" ); } /** Set the minDocCount on this builder, and return the builder so that calls can be chained. */ public IpPrefixAggregationBuilder minDocCount(long minDocCount) { if (minDocCount < 1) { - throwOnInvalidFieldValue( - MIN_DOC_COUNT_FIELD.getPreferredName(), - 1, - Integer.MAX_VALUE, - minDocCount - ); + throwOnInvalidFieldValue(MIN_DOC_COUNT_FIELD.getPreferredName(), 1, Integer.MAX_VALUE, minDocCount); } this.minDocCount = minDocCount; return this; @@ -214,7 +205,9 @@ protected ValuesSourceAggregatorFactory innerBuild( ) throws IOException { IpPrefixAggregationSupplier aggregationSupplier = context.getValuesSourceRegistry().getAggregator(REGISTRY_KEY, config); - if (prefixLength < 0 || (isIpv6 == false && prefixLength > IPV4_MAX_PREFIX_LENGTH) || (isIpv6 && prefixLength > IPV6_MAX_PREFIX_LENGTH)) { + if (prefixLength < 0 + || (isIpv6 == false && prefixLength > IPV4_MAX_PREFIX_LENGTH) + || (isIpv6 && prefixLength > IPV6_MAX_PREFIX_LENGTH)) { throwOnInvalidFieldValue( PREFIX_LENGTH_FIELD.getPreferredName(), MIN_PREFIX_LENGTH, @@ -255,7 +248,9 @@ protected ValuesSourceAggregatorFactory innerBuild( * network, or is not in range [0, 32] for an IPv4 network. */ public static BytesRef extractNetmask(int prefixLength, boolean isIpv6) { - if (prefixLength < 0 || (isIpv6 == false && prefixLength > IPV4_MAX_PREFIX_LENGTH) || (isIpv6 && prefixLength > IPV6_MAX_PREFIX_LENGTH)) { + if (prefixLength < 0 + || (isIpv6 == false && prefixLength > IPV4_MAX_PREFIX_LENGTH) + || (isIpv6 && prefixLength > IPV6_MAX_PREFIX_LENGTH)) { throwOnInvalidFieldValue( PREFIX_LENGTH_FIELD.getPreferredName(), MIN_PREFIX_LENGTH, @@ -270,14 +265,14 @@ public static BytesRef extractNetmask(int prefixLength, boolean isIpv6) { int bytesCount = prefixLength / 8; int bitsCount = prefixLength % 8; int i = 0; - //NOTE: first set whole bytes to 255 (0xFF) + // NOTE: first set whole bytes to 255 (0xFF) for (; i < bytesCount; i++) { ipAddress[i] = (byte) 0xFF; } - //NOTE: then set the remaining bits to 1. - //Trailing bits are already set to 0 at initialization time. - //Example: for prefixLength = 20, we first set 16 bits (2 bytes) - //to 0xFF, then set the remaining 4 bits to 1. + // NOTE: then set the remaining bits to 1. + // Trailing bits are already set to 0 at initialization time. + // Example: for prefixLength = 20, we first set 16 bits (2 bytes) + // to 0xFF, then set the remaining 4 bits to 1. if (bitsCount > 0) { int rem = 0; for (int j = 0; j < bitsCount; j++) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java index 58807902232ac..aa20d9f0e75ad 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationSupplier.java @@ -8,12 +8,10 @@ package org.elasticsearch.search.aggregations.bucket.prefix; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java index 097bef9697320..c3bbcf1efddab 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -164,9 +163,9 @@ private BytesRef maskIpAddress(final BytesRef ipAddress, final BytesRef netmask) } private BytesRef mask(final BytesRef ipAddress, final BytesRef subnetMask) { - //NOTE: IPv4 addresses are encoded as 16-bytes. As a result, we use an - //offset (12) to apply the subnet to the last 4 bytes (byes 12, 13, 14, 15) - //if the subnet mask is just a 4-bytes subnet mask. + // NOTE: IPv4 addresses are encoded as 16-bytes. As a result, we use an + // offset (12) to apply the subnet to the last 4 bytes (byes 12, 13, 14, 15) + // if the subnet mask is just a 4-bytes subnet mask. int offset = subnetMask.length == 4 ? 12 : 0; byte[] subnet = new byte[subnetMask.length]; for (int i = 0; i < subnetMask.length; ++i) { @@ -273,14 +272,7 @@ protected Unmapped( @Override public InternalAggregation buildEmptyAggregation() { - return new InternalIpPrefix( - name, - config.format(), - keyed, - minDocCount, - Collections.emptyList(), - metadata() - ); + return new InternalIpPrefix(name, config.format(), keyed, minDocCount, Collections.emptyList(), metadata()); } } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java index c953cb22cc8e5..ad14a2acbf6f1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorFactory.java @@ -52,32 +52,12 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { @Override protected Aggregator createUnmapped(Aggregator parent, Map metadata) throws IOException { - return new IpPrefixAggregator.Unmapped( - name, - factories, - config, - keyed, - minDocCount, - context, - parent, - metadata - ); + return new IpPrefixAggregator.Unmapped(name, factories, config, keyed, minDocCount, context, parent, metadata); } @Override protected Aggregator doCreateInternal(Aggregator parent, CardinalityUpperBound cardinality, Map metadata) throws IOException { - return aggregationSupplier.build( - name, - factories, - config, - keyed, - minDocCount, - ipPrefix, - context, - parent, - cardinality, - metadata - ); + return aggregationSupplier.build(name, factories, config, keyed, minDocCount, ipPrefix, context, parent, cardinality, metadata); } } From 3a967cd249fb957ab5989d0a6833c0c7cce3bf6a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 15:32:33 +0100 Subject: [PATCH 31/36] fix: error message for invalid prefix_length --- .../search/aggregations/bucket/IpPrefixTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java index 654f39f423bfa..d345731c1e22c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java @@ -35,7 +35,8 @@ public void testNegativePrefixLength() { final IpPrefixAggregationBuilder factory = new IpPrefixAggregationBuilder(randomAlphaOfLengthBetween(3, 10)); factory.isIpv6(randomBoolean()); - IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> factory.prefixLength(randomIntBetween(-1000, -1))); - assertThat(ex.getMessage(), startsWith("[prefix_len] must not be less than 0")); + int randomPrefixLength = randomIntBetween(-1000, -1); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> factory.prefixLength(randomPrefixLength)); + assertThat(ex.getMessage(), startsWith("[prefix_length] must be in range [0, 128] while value is [" + randomPrefixLength + "]")); } } From 83f12c4fcc6a1e7376ba68de782b9583bc2928fb Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 16:34:54 +0100 Subject: [PATCH 32/36] fix: use the parent testCase method --- .../prefix/IpPrefixAggregatorTests.java | 75 ++++++++++--------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java index 6296a9dd1dcf8..3654206ae3cad 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java @@ -140,7 +140,10 @@ public void testEmptyDocument() throws IOException { final List ipAddresses = Collections.emptyList(); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> {}, ipPrefix -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); @@ -185,13 +188,14 @@ public void testIpv4Addresses() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); @@ -234,13 +238,14 @@ public void testIpv6Addresses() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); @@ -284,13 +289,14 @@ public void testZeroPrefixLength() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); @@ -335,13 +341,14 @@ public void testIpv4MaxPrefixLength() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); @@ -384,13 +391,14 @@ public void testIpv6MaxPrefixLength() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); @@ -436,7 +444,7 @@ public void testAggregateOnIpv4Field() throws IOException { final String ipv6Value = "2001:db8:a4f8:112a:6001:0:12:7f2a"; // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( List.of( @@ -445,7 +453,8 @@ public void testAggregateOnIpv4Field() throws IOException { ) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); @@ -490,7 +499,7 @@ public void testAggregateOnIpv6Field() throws IOException { final String ipv4Value = "192.168.10.20"; // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( List.of( @@ -499,7 +508,8 @@ public void testAggregateOnIpv6Field() throws IOException { ) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); @@ -804,13 +814,14 @@ public void testIpv4AppendPrefixLength() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .map(appendPrefixLength(prefixLength)) @@ -855,13 +866,14 @@ public void testIpv6AppendPrefixLength() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .map(appendPrefixLength(prefixLength)) @@ -907,13 +919,14 @@ public void testMinDocCount() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( singleton(new SortedDocValuesField(field, new BytesRef(InetAddressPoint.encode(ipDataHolder.getIpAddress())))) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = Set.of("192.168.0.0"); final Set ipAddressesAsString = ipPrefix.getBuckets() .stream() @@ -961,7 +974,7 @@ public void testAggregationWithQueryFilter() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, query, iw -> { + testCase(aggregationBuilder, query, iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( List.of( @@ -970,7 +983,8 @@ public void testAggregationWithQueryFilter() throws IOException { ) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .filter(subnet -> subnet.startsWith("192.168.")) @@ -1020,7 +1034,7 @@ public void testMetricAggregation() throws IOException { ); // WHEN - testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { for (TestIpDataHolder ipDataHolder : ipAddresses) { iw.addDocument( List.of( @@ -1029,7 +1043,8 @@ public void testMetricAggregation() throws IOException { ) ); } - }, ipPrefix -> { + }, agg -> { + final InternalIpPrefix ipPrefix = (InternalIpPrefix) agg; final Set expectedSubnets = ipAddresses.stream() .map(TestIpDataHolder::getSubnetAsString) .collect(Collectors.toUnmodifiableSet()); @@ -1062,14 +1077,4 @@ private Function appendPrefixLength(int prefixLength) { private long defaultTime() { return randomLongBetween(0, Long.MAX_VALUE); } - - private void testAggregation( - AggregationBuilder aggregationBuilder, - Query query, - CheckedConsumer buildIndex, - Consumer verify, - MappedFieldType... fieldTypes - ) throws IOException { - testCase(aggregationBuilder, query, buildIndex, verify, fieldTypes); - } } From 3645a72037254766328a7bf9a20f7e7dbacc30fb Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 16:48:19 +0100 Subject: [PATCH 33/36] fix: code format violations --- .../aggregations/bucket/prefix/IpPrefixAggregatorTests.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java index 3654206ae3cad..8b032e1690923 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregatorTests.java @@ -12,12 +12,10 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.network.InetAddresses; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -38,7 +36,6 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; From c4d20e69f37208e954e5c455fda5d800c64feae4 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 20 Jan 2022 17:12:39 +0100 Subject: [PATCH 34/36] fix: use the correct prefix length range depending on the isIpv6 random boolean --- .../search/aggregations/bucket/IpPrefixTests.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java index d345731c1e22c..7b7602ec67816 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpPrefixTests.java @@ -33,10 +33,15 @@ protected IpPrefixAggregationBuilder createTestAggregatorBuilder() { public void testNegativePrefixLength() { final IpPrefixAggregationBuilder factory = new IpPrefixAggregationBuilder(randomAlphaOfLengthBetween(3, 10)); - factory.isIpv6(randomBoolean()); - + boolean isIpv6 = randomBoolean(); + final String rangeAsString = isIpv6 ? "[0, 128]" : "[0, 32]"; + factory.isIpv6(isIpv6); int randomPrefixLength = randomIntBetween(-1000, -1); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> factory.prefixLength(randomPrefixLength)); - assertThat(ex.getMessage(), startsWith("[prefix_length] must be in range [0, 128] while value is [" + randomPrefixLength + "]")); + assertThat( + ex.getMessage(), + startsWith("[prefix_length] must be in range " + rangeAsString + " while value is [" + randomPrefixLength + "]") + ); } } From 658a844e52e3fa156ba6db0d39401b2889b8542e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Wed, 26 Jan 2022 15:54:56 +0100 Subject: [PATCH 35/36] fix: re-use the subnet buffer instead of re-allocating byte arrays --- .../bucket/prefix/IpPrefixAggregator.java | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java index c3bbcf1efddab..a195e48feccef 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregator.java @@ -114,9 +114,7 @@ public IpPrefixAggregator( @Override protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { - return config.getValuesSource() == null - ? LeafBucketCollector.NO_OP_COLLECTOR - : new IpPrefixLeafCollector(sub, config.getValuesSource().bytesValues(ctx), ipPrefix); + return new IpPrefixLeafCollector(sub, config.getValuesSource().bytesValues(ctx), ipPrefix); } private class IpPrefixLeafCollector extends LeafBucketCollectorBase { @@ -124,7 +122,7 @@ private class IpPrefixLeafCollector extends LeafBucketCollectorBase { private final LeafBucketCollector sub; private final SortedBinaryDocValues values; - IpPrefixLeafCollector(LeafBucketCollector sub, SortedBinaryDocValues values, IpPrefix ipPrefix) { + IpPrefixLeafCollector(final LeafBucketCollector sub, final SortedBinaryDocValues values, final IpPrefix ipPrefix) { super(sub, values); this.sub = sub; this.values = values; @@ -134,14 +132,14 @@ private class IpPrefixLeafCollector extends LeafBucketCollectorBase { @Override public void collect(int doc, long owningBucketOrd) throws IOException { BytesRef previousSubnet = null; - BytesRef subnet; + BytesRef subnet = new BytesRef(new byte[ipPrefix.netmask.length]); BytesRef ipAddress; if (values.advanceExact(doc)) { int valuesCount = values.docValueCount(); for (int i = 0; i < valuesCount; ++i) { ipAddress = values.nextValue(); - subnet = maskIpAddress(ipAddress, ipPrefix.netmask); + maskIpAddress(ipAddress, ipPrefix.netmask, subnet); if (previousSubnet != null && subnet.bytesEquals(previousSubnet)) { continue; } @@ -157,22 +155,15 @@ public void collect(int doc, long owningBucketOrd) throws IOException { } } - private BytesRef maskIpAddress(final BytesRef ipAddress, final BytesRef netmask) { + private void maskIpAddress(final BytesRef ipAddress, final BytesRef subnetMask, final BytesRef subnet) { assert ipAddress.length == 16 : "Invalid length for ip address [" + ipAddress.length + "] expected 16 bytes"; - return mask(ipAddress, netmask); - } - - private BytesRef mask(final BytesRef ipAddress, final BytesRef subnetMask) { // NOTE: IPv4 addresses are encoded as 16-bytes. As a result, we use an // offset (12) to apply the subnet to the last 4 bytes (byes 12, 13, 14, 15) // if the subnet mask is just a 4-bytes subnet mask. int offset = subnetMask.length == 4 ? 12 : 0; - byte[] subnet = new byte[subnetMask.length]; for (int i = 0; i < subnetMask.length; ++i) { - subnet[i] = (byte) (ipAddress.bytes[i + offset] & subnetMask.bytes[i]); + subnet.bytes[i] = (byte) (ipAddress.bytes[i + offset] & subnetMask.bytes[i]); } - - return new BytesRef(subnet); } } From e4517a2b95d86e368ab6358c31ba04545d58a192 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Wed, 26 Jan 2022 16:01:32 +0100 Subject: [PATCH 36/36] Update docs/changelog/82410.yaml --- docs/changelog/82410.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/82410.yaml diff --git a/docs/changelog/82410.yaml b/docs/changelog/82410.yaml new file mode 100644 index 0000000000000..b21ebe77f30a3 --- /dev/null +++ b/docs/changelog/82410.yaml @@ -0,0 +1,5 @@ +pr: 82410 +summary: Add an aggregator for IPv4 and IPv6 subnets +area: Aggregations +type: enhancement +issues: []