From ee1648b81d96ccdd9d699ab294dccb403d976dc6 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Mon, 1 Apr 2019 16:09:54 -0400 Subject: [PATCH 01/13] Double range decoder --- .../queries/BinaryDocValuesRangeQuery.java | 10 +++---- .../index/mapper/BinaryRangeUtil.java | 30 +++++++++++++++++++ .../index/mapper/RangeFieldMapper.java | 21 +++++++++++++ .../index/mapper/BinaryRangeUtilTests.java | 23 ++++++++++++++ 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java index 9fa9bcacc0ac6..73de31f2a97e0 100644 --- a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -181,25 +181,25 @@ boolean matches(BytesRef from, BytesRef to, BytesRef otherFrom, BytesRef otherTo public enum LengthType { FIXED_4 { @Override - int readLength(byte[] bytes, int offset) { + public int readLength(byte[] bytes, int offset) { return 4; } }, FIXED_8 { @Override - int readLength(byte[] bytes, int offset) { + public int readLength(byte[] bytes, int offset) { return 8; } }, FIXED_16 { @Override - int readLength(byte[] bytes, int offset) { + public int readLength(byte[] bytes, int offset) { return 16; } }, VARIABLE { @Override - int readLength(byte[] bytes, int offset) { + public int readLength(byte[] bytes, int offset) { // the first bit encodes the sign and the next 4 bits encode the number // of additional bytes int token = Byte.toUnsignedInt(bytes[offset]); @@ -214,6 +214,6 @@ int readLength(byte[] bytes, int offset) { /** * Return the length of the value that starts at {@code offset} in {@code bytes}. */ - abstract int readLength(byte[] bytes, int offset); + public abstract int readLength(byte[] bytes, int offset); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index b58c6deba8cd3..106b892b82e19 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.queries.BinaryDocValuesRangeQuery; +import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; @@ -51,6 +53,10 @@ static BytesRef encodeLongRanges(Set ranges) throws IOEx return new BytesRef(encoded, 0, out.getPosition()); } + static List decodeLongRanges(BytesRef encodedRanges) { + return null; + } + static BytesRef encodeDoubleRanges(Set ranges) throws IOException { List sortedRanges = new ArrayList<>(ranges); Comparator fromComparator = Comparator.comparingDouble(range -> ((Number) range.from).doubleValue()); @@ -69,6 +75,30 @@ static BytesRef encodeDoubleRanges(Set ranges) throws IO return new BytesRef(encoded, 0, out.getPosition()); } + static List decodeDoubleRanges(BytesRef encodedRanges) { + BinaryDocValuesRangeQuery.LengthType lengthType = BinaryDocValuesRangeQuery.LengthType.FIXED_8; + + ByteArrayDataInput in = new ByteArrayDataInput(); + in.reset(encodedRanges.bytes, encodedRanges.offset, encodedRanges.length); + int numRanges = in.readVInt(); + + List ranges = new ArrayList<>(numRanges); + + final byte[] bytes = encodedRanges.bytes; + int offset = in.getPosition(); + for (int i = 0; i < numRanges; i++) { + int length = lengthType.readLength(bytes, offset); + double from = NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, offset)); + offset += length; + + length = lengthType.readLength(bytes, offset); + double to = NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, offset)); + offset += length; + ranges.add(new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, from, to, true, true)); + } + return ranges; + } + static BytesRef encodeFloatRanges(Set ranges) throws IOException { List sortedRanges = new ArrayList<>(ranges); Comparator fromComparator = Comparator.comparingDouble(range -> ((Number) range.from).floatValue()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index e5ba55de7bfd0..c58018b41bdce 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -969,6 +969,27 @@ public Range(RangeType type, Object from, Object to, boolean includeFrom, boolea this.includeTo = includeTo; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Range range = (Range) o; + return includeFrom == range.includeFrom && + includeTo == range.includeTo && + type == range.type && + from.equals(range.from) && + to.equals(range.to); + } + + @Override + public int hashCode() { + return Objects.hash(type, from, to, includeFrom, includeTo); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index 20d4af1f0b600..964dc901ae173 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -21,6 +21,11 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.singleton; + public class BinaryRangeUtilTests extends ESTestCase { public void testBasics() { @@ -140,6 +145,24 @@ public void testEncode_Float() { } } + public void testDecodeLongRanges() throws IOException { + // TODO: Apply randomized testing here + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.LONG, -10, 42, true, true); + List decoded = BinaryRangeUtil.decodeLongRanges(BinaryRangeUtil.encodeLongRanges(singleton(expected))); + assertEquals(1, decoded.size()); + RangeFieldMapper.Range actual = decoded.get(1); + assertEquals(expected, actual); + } + + public void testDecodeDoubleRanges() throws IOException { + // TODO: Apply randomized testing here + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, -10.0D, 42.3D, true, true); + List decoded = BinaryRangeUtil.decodeDoubleRanges(BinaryRangeUtil.encodeDoubleRanges(singleton(expected))); + assertEquals(1, decoded.size()); + RangeFieldMapper.Range actual = decoded.get(0); + assertEquals(expected, actual); + } + private static int normalize(int cmp) { if (cmp < 0) { return -1; From a302f01fe0be346c121cca618e4cf8e3664d0edb Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Wed, 3 Apr 2019 11:20:50 -0400 Subject: [PATCH 02/13] refactoring the decode function --- .../index/mapper/BinaryRangeUtil.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index 106b892b82e19..1e998045bcca2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -24,6 +24,7 @@ import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.TriFunction; import java.io.IOException; import java.util.ArrayList; @@ -75,8 +76,12 @@ static BytesRef encodeDoubleRanges(Set ranges) throws IO return new BytesRef(encoded, 0, out.getPosition()); } - static List decodeDoubleRanges(BytesRef encodedRanges) { - BinaryDocValuesRangeQuery.LengthType lengthType = BinaryDocValuesRangeQuery.LengthType.FIXED_8; + static List decodeDoubleRanges(BytesRef encodedRanges) { + return decodeRanges(encodedRanges, BinaryDocValuesRangeQuery.LengthType.FIXED_8, BinaryRangeUtil::decodeDouble); + } + + static List decodeRanges(BytesRef encodedRanges, BinaryDocValuesRangeQuery.LengthType lengthType, + TriFunction decodeBytes) { ByteArrayDataInput in = new ByteArrayDataInput(); in.reset(encodedRanges.bytes, encodedRanges.offset, encodedRanges.length); @@ -87,14 +92,17 @@ static List decodeDoubleRanges(BytesRef encodedRanges) final byte[] bytes = encodedRanges.bytes; int offset = in.getPosition(); for (int i = 0; i < numRanges; i++) { + // TODO: make this a lambda parmeter int length = lengthType.readLength(bytes, offset); - double from = NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, offset)); + Object from = decodeBytes.apply(bytes, offset, length); offset += length; length = lengthType.readLength(bytes, offset); - double to = NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, offset)); + Object to = decodeBytes.apply(bytes, offset, length); offset += length; - ranges.add(new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, from, to, true, true)); + // TODO: Support for exclusive ranges, pending resolution of #40601 + RangeFieldMapper.Range decodedRange = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, from, to, true, true); + ranges.add(decodedRange); } return ranges; } @@ -123,6 +131,10 @@ static byte[] encodeDouble(double number) { return encoded; } + static double decodeDouble(byte[] bytes, int offset, int length){ + return NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, offset)); + } + static byte[] encodeFloat(float number) { byte[] encoded = new byte[4]; NumericUtils.intToSortableBytes(NumericUtils.floatToSortableInt(number), encoded, 0); From 591d26bbf4708fb5c0b05316550a2f5ddc7e545c Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Wed, 3 Apr 2019 11:49:15 -0400 Subject: [PATCH 03/13] decode float ranges --- .../index/mapper/BinaryRangeUtil.java | 18 ++++++++++++++---- .../index/mapper/BinaryRangeUtilTests.java | 9 +++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index 1e998045bcca2..c624a1f8e1e4a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -77,10 +77,17 @@ static BytesRef encodeDoubleRanges(Set ranges) throws IO } static List decodeDoubleRanges(BytesRef encodedRanges) { - return decodeRanges(encodedRanges, BinaryDocValuesRangeQuery.LengthType.FIXED_8, BinaryRangeUtil::decodeDouble); + return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.DOUBLE, BinaryDocValuesRangeQuery.LengthType.FIXED_8, + BinaryRangeUtil::decodeDouble); } - static List decodeRanges(BytesRef encodedRanges, BinaryDocValuesRangeQuery.LengthType lengthType, + static List decodeFloatRanges(BytesRef encodedRanges) { + return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.FLOAT, BinaryDocValuesRangeQuery.LengthType.FIXED_4, + BinaryRangeUtil::decodeFloat); + } + + static List decodeRanges(BytesRef encodedRanges, RangeFieldMapper.RangeType rangeType, + BinaryDocValuesRangeQuery.LengthType lengthType, TriFunction decodeBytes) { ByteArrayDataInput in = new ByteArrayDataInput(); @@ -92,7 +99,6 @@ static List decodeRanges(BytesRef encodedRanges, BinaryD final byte[] bytes = encodedRanges.bytes; int offset = in.getPosition(); for (int i = 0; i < numRanges; i++) { - // TODO: make this a lambda parmeter int length = lengthType.readLength(bytes, offset); Object from = decodeBytes.apply(bytes, offset, length); offset += length; @@ -101,7 +107,7 @@ static List decodeRanges(BytesRef encodedRanges, BinaryD Object to = decodeBytes.apply(bytes, offset, length); offset += length; // TODO: Support for exclusive ranges, pending resolution of #40601 - RangeFieldMapper.Range decodedRange = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, from, to, true, true); + RangeFieldMapper.Range decodedRange = new RangeFieldMapper.Range(rangeType, from, to, true, true); ranges.add(decodedRange); } return ranges; @@ -141,6 +147,10 @@ static byte[] encodeFloat(float number) { return encoded; } + static float decodeFloat(byte[] bytes, int offset, int length) { + return NumericUtils.sortableIntToFloat(NumericUtils.sortableBytesToInt(bytes, offset)); + } + /** * Encodes the specified number of type long in a variable-length byte format. * The byte format preserves ordering, which means the returned byte array can be used for comparing as is. diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index 964dc901ae173..f54551dcce5f6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -163,6 +163,15 @@ public void testDecodeDoubleRanges() throws IOException { assertEquals(expected, actual); } + public void testDecodeFloatRanges() throws IOException { + // TODO: Apply randomized testing here + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.FLOAT, -10.0F, 42.3F, true, true); + List decoded = BinaryRangeUtil.decodeFloatRanges(BinaryRangeUtil.encodeFloatRanges(singleton(expected))); + assertEquals(1, decoded.size()); + RangeFieldMapper.Range actual = decoded.get(0); + assertEquals(expected, actual); + } + private static int normalize(int cmp) { if (cmp < 0) { return -1; From db2b57651ed808c1d9ee97c0616150b733cb7826 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 4 Apr 2019 13:12:48 -0400 Subject: [PATCH 04/13] decode long ranges --- .../index/mapper/BinaryRangeUtil.java | 20 ++++++++++++++++++- .../index/mapper/BinaryRangeUtilTests.java | 15 ++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index c624a1f8e1e4a..8220d9bddbb33 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -55,7 +55,8 @@ static BytesRef encodeLongRanges(Set ranges) throws IOEx } static List decodeLongRanges(BytesRef encodedRanges) { - return null; + return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.LONG, BinaryDocValuesRangeQuery.LengthType.VARIABLE, + BinaryRangeUtil::decodeLong); } static BytesRef encodeDoubleRanges(Set ranges) throws IOException { @@ -166,6 +167,23 @@ static byte[] encodeLong(long number) { return encode(number, sign); } + static long decodeLong(byte[] bytes, int offset, int length) { + boolean isNegative = (bytes[offset] & 128) == 0; + // Start by masking off the last three bits of the first byte - that's the start of our number + long decoded; + if (isNegative) { + decoded = -8 | bytes[offset]; + } else { + decoded = bytes[offset] & 7; + } + for (int i = 1; i < length; i++) { + decoded <<= 8; + decoded += Byte.toUnsignedInt(bytes[offset + i]); + } + + return decoded; + } + private static byte[] encode(long l, int sign) { assert l >= 0; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index f54551dcce5f6..7a20e6e943a66 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.queries.BinaryDocValuesRangeQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; @@ -145,12 +146,22 @@ public void testEncode_Float() { } } + public void testDecodeLong() { + long[] cases = new long[] { Long.MIN_VALUE, -1, -128, -3, 0, 3, 125, 2048, 2049, Long.MAX_VALUE}; + for (long expected : cases) { + byte[] encoded = BinaryRangeUtil.encodeLong(expected); + int offset = 0; + int length = BinaryDocValuesRangeQuery.LengthType.VARIABLE.readLength(encoded, offset); + assertEquals(expected, BinaryRangeUtil.decodeLong(encoded, offset, length)); + } + } + public void testDecodeLongRanges() throws IOException { // TODO: Apply randomized testing here - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.LONG, -10, 42, true, true); + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.LONG, -10L, 42L, true, true); List decoded = BinaryRangeUtil.decodeLongRanges(BinaryRangeUtil.encodeLongRanges(singleton(expected))); assertEquals(1, decoded.size()); - RangeFieldMapper.Range actual = decoded.get(1); + RangeFieldMapper.Range actual = decoded.get(0); assertEquals(expected, actual); } From 8b9c50a6b777c2870bd1826f82c55fd9c4356240 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 4 Apr 2019 14:31:46 -0400 Subject: [PATCH 05/13] Make lengthType a property of RangeType --- .../index/mapper/BinaryRangeUtil.java | 8 +++---- .../index/mapper/RangeFieldMapper.java | 22 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index 8220d9bddbb33..b362d10ed2893 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -55,7 +55,7 @@ static BytesRef encodeLongRanges(Set ranges) throws IOEx } static List decodeLongRanges(BytesRef encodedRanges) { - return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.LONG, BinaryDocValuesRangeQuery.LengthType.VARIABLE, + return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.LONG, BinaryRangeUtil::decodeLong); } @@ -78,19 +78,19 @@ static BytesRef encodeDoubleRanges(Set ranges) throws IO } static List decodeDoubleRanges(BytesRef encodedRanges) { - return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.DOUBLE, BinaryDocValuesRangeQuery.LengthType.FIXED_8, + return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.DOUBLE, BinaryRangeUtil::decodeDouble); } static List decodeFloatRanges(BytesRef encodedRanges) { - return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.FLOAT, BinaryDocValuesRangeQuery.LengthType.FIXED_4, + return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.FLOAT, BinaryRangeUtil::decodeFloat); } static List decodeRanges(BytesRef encodedRanges, RangeFieldMapper.RangeType rangeType, - BinaryDocValuesRangeQuery.LengthType lengthType, TriFunction decodeBytes) { + BinaryDocValuesRangeQuery.LengthType lengthType = rangeType.lengthType; ByteArrayDataInput in = new ByteArrayDataInput(); in.reset(encodedRanges.bytes, encodedRanges.offset, encodedRanges.length); int numRanges = in.readVInt(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index c58018b41bdce..418f4b7c12258 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -437,7 +437,7 @@ private static Range parseIpRangeFromCidr(final XContentParser parser) throws IO /** Enum defining the type of range */ public enum RangeType { - IP("ip_range") { + IP("ip_range", BinaryDocValuesRangeQuery.LengthType.FIXED_16) { @Override public Field getRangeField(String name, Range r) { return new InetAddressRange(name, (InetAddress)r.from, (InetAddress)r.to); @@ -537,7 +537,7 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu includeLower ? lower : nextUp(lower), includeUpper ? upper : nextDown(upper)); } }, - DATE("date_range", NumberType.LONG) { + DATE("date_range", BinaryDocValuesRangeQuery.LengthType.VARIABLE, NumberType.LONG) { @Override public Field getRangeField(String name, Range r) { return new LongRange(name, new long[] {((Number)r.from).longValue()}, new long[] {((Number)r.to).longValue()}); @@ -617,7 +617,7 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu } }, // todo support half_float - FLOAT("float_range", NumberType.FLOAT) { + FLOAT("float_range", BinaryDocValuesRangeQuery.LengthType.FIXED_4, NumberType.FLOAT) { @Override public Float minValue() { return Float.NEGATIVE_INFINITY; @@ -679,7 +679,7 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu new float[] {includeTo ? (Float)to : Math.nextDown((Float)to)}); } }, - DOUBLE("double_range", NumberType.DOUBLE) { + DOUBLE("double_range", BinaryDocValuesRangeQuery.LengthType.FIXED_8, NumberType.DOUBLE) { @Override public Double minValue() { return Double.NEGATIVE_INFINITY; @@ -743,7 +743,7 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu }, // todo add BYTE support // todo add SHORT support - INTEGER("integer_range", NumberType.INTEGER) { + INTEGER("integer_range", BinaryDocValuesRangeQuery.LengthType.VARIABLE, NumberType.INTEGER) { @Override public Integer minValue() { return Integer.MIN_VALUE; @@ -791,7 +791,7 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu new int[] {(Integer)to - (includeTo ? 0 : 1)}); } }, - LONG("long_range", NumberType.LONG) { + LONG("long_range", BinaryDocValuesRangeQuery.LengthType.VARIABLE, NumberType.LONG) { @Override public Long minValue() { return Long.MIN_VALUE; @@ -852,14 +852,16 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu } }; - RangeType(String name) { + RangeType(String name, BinaryDocValuesRangeQuery.LengthType lengthType) { this.name = name; this.numberType = null; + this.lengthType = lengthType; } - RangeType(String name, NumberType type) { + RangeType(String name, BinaryDocValuesRangeQuery.LengthType lengthType, NumberType type) { this.name = name; this.numberType = type; + this.lengthType = lengthType; } /** Get the associated type name. */ @@ -948,9 +950,7 @@ public abstract Query dvRangeQuery(String field, QueryType queryType, Object fro public final String name; private final NumberType numberType; - - - + public final BinaryDocValuesRangeQuery.LengthType lengthType; } /** Class defining a range */ From 48e8b55f949805caa9ce6bc21dab31ca81819baf Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 4 Apr 2019 16:01:04 -0400 Subject: [PATCH 06/13] wire decode logic into the RangeType enum --- .../index/mapper/RangeFieldMapper.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 418f4b7c12258..60d1a51e00f0b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -499,6 +499,12 @@ public BytesRef encodeRanges(Set ranges) throws IOException { return new BytesRef(encoded, 0, out.getPosition()); } + @Override + public List decodeRanges(BytesRef bytes) { + // TODO: Implement this. + throw new UnsupportedOperationException(); + } + @Override public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { if (includeFrom == false) { @@ -580,6 +586,11 @@ public BytesRef encodeRanges(Set ranges) throws IOException { return LONG.encodeRanges(ranges); } + @Override + public List decodeRanges(BytesRef bytes) { + return LONG.decodeRanges(bytes); + } + @Override public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); @@ -640,6 +651,11 @@ public BytesRef encodeRanges(Set ranges) throws IOException { return BinaryRangeUtil.encodeFloatRanges(ranges); } + @Override + public List decodeRanges(BytesRef bytes) { + return BinaryRangeUtil.decodeFloatRanges(bytes); + } + @Override public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { if (includeFrom == false) { @@ -702,6 +718,11 @@ public BytesRef encodeRanges(Set ranges) throws IOException { return BinaryRangeUtil.encodeDoubleRanges(ranges); } + @Override + public List decodeRanges(BytesRef bytes) { + return BinaryRangeUtil.decodeDoubleRanges(bytes); + } + @Override public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { if (includeFrom == false) { @@ -766,6 +787,11 @@ public BytesRef encodeRanges(Set ranges) throws IOException { return LONG.encodeRanges(ranges); } + @Override + public List decodeRanges(BytesRef bytes) { + return LONG.decodeRanges(bytes); + } + @Override public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); @@ -814,6 +840,11 @@ public BytesRef encodeRanges(Set ranges) throws IOException { return BinaryRangeUtil.encodeLongRanges(ranges); } + @Override + public List decodeRanges(BytesRef bytes) { + return BinaryRangeUtil.decodeLongRanges(bytes); + } + @Override public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { if (includeFrom == false) { @@ -944,6 +975,7 @@ public Query rangeQuery(String field, boolean hasDocValues, Object from, Object // No need to take into account Range#includeFrom or Range#includeTo, because from and to have already been // rounded up via parseFrom and parseTo methods. public abstract BytesRef encodeRanges(Set ranges) throws IOException; + public abstract List decodeRanges(BytesRef bytes); public abstract Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo); From 99881710a55aa16ecec5a3907f146035ec90ab44 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 11 Apr 2019 12:03:37 -0400 Subject: [PATCH 07/13] IP range decoder & tests --- .../index/mapper/BinaryRangeUtil.java | 29 +++++++++++++++++++ .../index/mapper/RangeFieldMapper.java | 15 +--------- .../index/mapper/BinaryRangeUtilTests.java | 10 +++++++ 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index b362d10ed2893..3a87394db91c4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.queries.BinaryDocValuesRangeQuery; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteArrayDataOutput; @@ -27,7 +28,9 @@ import org.elasticsearch.common.TriFunction; import java.io.IOException; +import java.net.InetAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Set; @@ -36,6 +39,32 @@ enum BinaryRangeUtil { ; + static BytesRef encodeIPRanges(Set ranges) throws IOException { + final byte[] encoded = new byte[5 + (16 * 2) * ranges.size()]; + ByteArrayDataOutput out = new ByteArrayDataOutput(encoded); + out.writeVInt(ranges.size()); + for (RangeFieldMapper.Range range : ranges) { + InetAddress fromValue = (InetAddress) range.from; + byte[] encodedFromValue = InetAddressPoint.encode(fromValue); + out.writeBytes(encodedFromValue, 0, encodedFromValue.length); + + InetAddress toValue = (InetAddress) range.to; + byte[] encodedToValue = InetAddressPoint.encode(toValue); + out.writeBytes(encodedToValue, 0, encodedToValue.length); + } + return new BytesRef(encoded, 0, out.getPosition()); + } + + static List decodeIPRanges(BytesRef encodedRanges) { + return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.IP, BinaryRangeUtil::decodeIP); + } + + static private InetAddress decodeIP(byte[] bytes, int offset, int length) { + // offset + length because copyOfRange wants a from and a to, not an offset & length + byte[] slice = Arrays.copyOfRange(bytes, offset, offset + length); + return InetAddressPoint.decode(slice); + } + static BytesRef encodeLongRanges(Set ranges) throws IOException { List sortedRanges = new ArrayList<>(ranges); Comparator fromComparator = Comparator.comparingLong(range -> ((Number) range.from).longValue()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 60d1a51e00f0b..53f805b064f77 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -37,7 +37,6 @@ import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Explicit; @@ -484,19 +483,7 @@ public InetAddress nextDown(Object value) { @Override public BytesRef encodeRanges(Set ranges) throws IOException { - final byte[] encoded = new byte[5 + (16 * 2) * ranges.size()]; - ByteArrayDataOutput out = new ByteArrayDataOutput(encoded); - out.writeVInt(ranges.size()); - for (Range range : ranges) { - InetAddress fromValue = (InetAddress) range.from; - byte[] encodedFromValue = InetAddressPoint.encode(fromValue); - out.writeBytes(encodedFromValue, 0, encodedFromValue.length); - - InetAddress toValue = (InetAddress) range.to; - byte[] encodedToValue = InetAddressPoint.encode(toValue); - out.writeBytes(encodedToValue, 0, encodedToValue.length); - } - return new BytesRef(encoded, 0, out.getPosition()); + return BinaryRangeUtil.encodeIPRanges(ranges); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index 7a20e6e943a66..60f5351b12a93 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -20,6 +20,7 @@ import org.apache.lucene.queries.BinaryDocValuesRangeQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -183,6 +184,15 @@ public void testDecodeFloatRanges() throws IOException { assertEquals(expected, actual); } + public void testDecodeIPRanges() throws IOException { + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.IP, InetAddresses.forString("192.168.0.1"), + InetAddresses.forString("192.168.0.100"), true, true); + List decoded = BinaryRangeUtil.decodeIPRanges(BinaryRangeUtil.encodeIPRanges(singleton(expected))); + assertEquals(1, decoded.size()); + RangeFieldMapper.Range actual = decoded.get(0); + assertEquals(expected, actual); + } + private static int normalize(int cmp) { if (cmp < 0) { return -1; From fd1f5d956aaaa3e78ac89ea5a31aa375fd6b1053 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Mon, 15 Apr 2019 10:19:04 -0400 Subject: [PATCH 08/13] Some test randomization --- .../index/mapper/BinaryRangeUtilTests.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index 60f5351b12a93..6fd3fa26b595a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -148,7 +148,7 @@ public void testEncode_Float() { } public void testDecodeLong() { - long[] cases = new long[] { Long.MIN_VALUE, -1, -128, -3, 0, 3, 125, 2048, 2049, Long.MAX_VALUE}; + long[] cases = new long[] { Long.MIN_VALUE, -2049, -2048, -128, -3, -1, 0, 1, 3, 125, 2048, 2049, Long.MAX_VALUE}; for (long expected : cases) { byte[] encoded = BinaryRangeUtil.encodeLong(expected); int offset = 0; @@ -158,8 +158,9 @@ public void testDecodeLong() { } public void testDecodeLongRanges() throws IOException { - // TODO: Apply randomized testing here - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.LONG, -10L, 42L, true, true); + long start = randomLong(); + long end = randomLongBetween(start + 1, Long.MAX_VALUE); + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.LONG, start, end, true, true); List decoded = BinaryRangeUtil.decodeLongRanges(BinaryRangeUtil.encodeLongRanges(singleton(expected))); assertEquals(1, decoded.size()); RangeFieldMapper.Range actual = decoded.get(0); @@ -167,8 +168,9 @@ public void testDecodeLongRanges() throws IOException { } public void testDecodeDoubleRanges() throws IOException { - // TODO: Apply randomized testing here - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, -10.0D, 42.3D, true, true); + double start = randomDouble(); + double end = randomDoubleBetween(Math.nextUp(start), Double.MAX_VALUE, false); + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, start, end, true, true); List decoded = BinaryRangeUtil.decodeDoubleRanges(BinaryRangeUtil.encodeDoubleRanges(singleton(expected))); assertEquals(1, decoded.size()); RangeFieldMapper.Range actual = decoded.get(0); @@ -176,8 +178,15 @@ public void testDecodeDoubleRanges() throws IOException { } public void testDecodeFloatRanges() throws IOException { - // TODO: Apply randomized testing here - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.FLOAT, -10.0F, 42.3F, true, true); + float start = randomFloat(); + // for some reason, ESTestCase doesn't provide randomFloatBetween + float end = randomFloat(); + if (start > end) { + float temp = start; + start = end; + end = temp; + } + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.FLOAT, start, end, true, true); List decoded = BinaryRangeUtil.decodeFloatRanges(BinaryRangeUtil.encodeFloatRanges(singleton(expected))); assertEquals(1, decoded.size()); RangeFieldMapper.Range actual = decoded.get(0); From fd39453266433b68a513e1bb2d3215a22bff9f09 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Mon, 15 Apr 2019 10:51:06 -0400 Subject: [PATCH 09/13] fixed checkstyle issue --- .../java/org/elasticsearch/index/mapper/BinaryRangeUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index 3a87394db91c4..764befd2b0460 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -59,7 +59,7 @@ static List decodeIPRanges(BytesRef encodedRanges) { return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.IP, BinaryRangeUtil::decodeIP); } - static private InetAddress decodeIP(byte[] bytes, int offset, int length) { + private static InetAddress decodeIP(byte[] bytes, int offset, int length) { // offset + length because copyOfRange wants a from and a to, not an offset & length byte[] slice = Arrays.copyOfRange(bytes, offset, offset + length); return InetAddressPoint.decode(slice); From 9492559557368bd7d41210f75556ed2ece53a322 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Wed, 17 Apr 2019 12:55:00 -0400 Subject: [PATCH 10/13] refactored LengthType into BinaryRangeUtil --- .../queries/BinaryDocValuesRangeQuery.java | 43 ++---------------- .../index/mapper/BinaryRangeUtil.java | 44 +++++++++++++++++-- .../index/mapper/RangeFieldMapper.java | 26 +++++------ .../index/mapper/BinaryRangeUtilTests.java | 3 +- 4 files changed, 58 insertions(+), 58 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java index 73de31f2a97e0..94201a72aae00 100644 --- a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.BinaryRangeUtil; import java.io.IOException; import java.util.Objects; @@ -40,13 +41,13 @@ public final class BinaryDocValuesRangeQuery extends Query { private final String fieldName; private final QueryType queryType; - private final LengthType lengthType; + private final BinaryRangeUtil.LengthType lengthType; private final BytesRef from; private final BytesRef to; private final Object originalFrom; private final Object originalTo; - public BinaryDocValuesRangeQuery(String fieldName, QueryType queryType, LengthType lengthType, + public BinaryDocValuesRangeQuery(String fieldName, QueryType queryType, BinaryRangeUtil.LengthType lengthType, BytesRef from, BytesRef to, Object originalFrom, Object originalTo) { this.fieldName = fieldName; @@ -178,42 +179,4 @@ boolean matches(BytesRef from, BytesRef to, BytesRef otherFrom, BytesRef otherTo } - public enum LengthType { - FIXED_4 { - @Override - public int readLength(byte[] bytes, int offset) { - return 4; - } - }, - FIXED_8 { - @Override - public int readLength(byte[] bytes, int offset) { - return 8; - } - }, - FIXED_16 { - @Override - public int readLength(byte[] bytes, int offset) { - return 16; - } - }, - VARIABLE { - @Override - public int readLength(byte[] bytes, int offset) { - // the first bit encodes the sign and the next 4 bits encode the number - // of additional bytes - int token = Byte.toUnsignedInt(bytes[offset]); - int length = (token >>> 3) & 0x0f; - if ((token & 0x80) == 0) { - length = 0x0f - length; - } - return 1 + length; - } - }; - - /** - * Return the length of the value that starts at {@code offset} in {@code bytes}. - */ - public abstract int readLength(byte[] bytes, int offset); - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index 764befd2b0460..b387d69b3cc3e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.InetAddressPoint; -import org.apache.lucene.queries.BinaryDocValuesRangeQuery; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.BytesRef; @@ -35,7 +34,7 @@ import java.util.List; import java.util.Set; -enum BinaryRangeUtil { +public enum BinaryRangeUtil { ; @@ -119,7 +118,7 @@ static List decodeFloatRanges(BytesRef encodedRanges) { static List decodeRanges(BytesRef encodedRanges, RangeFieldMapper.RangeType rangeType, TriFunction decodeBytes) { - BinaryDocValuesRangeQuery.LengthType lengthType = rangeType.lengthType; + LengthType lengthType = rangeType.lengthType; ByteArrayDataInput in = new ByteArrayDataInput(); in.reset(encodedRanges.bytes, encodedRanges.offset, encodedRanges.length); int numRanges = in.readVInt(); @@ -257,4 +256,43 @@ private static byte[] encode(long l, int sign) { } return encoded; } + + public enum LengthType { + FIXED_4 { + @Override + public int readLength(byte[] bytes, int offset) { + return 4; + } + }, + FIXED_8 { + @Override + public int readLength(byte[] bytes, int offset) { + return 8; + } + }, + FIXED_16 { + @Override + public int readLength(byte[] bytes, int offset) { + return 16; + } + }, + VARIABLE { + @Override + public int readLength(byte[] bytes, int offset) { + // the first bit encodes the sign and the next 4 bits encode the number + // of additional bytes + int token = Byte.toUnsignedInt(bytes[offset]); + int length = (token >>> 3) & 0x0f; + if ((token & 0x80) == 0) { + length = 0x0f - length; + } + return 1 + length; + } + }; + + /** + * Return the length of the value that starts at {@code offset} in {@code bytes}. + */ + public abstract int readLength(byte[] bytes, int offset); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 96b699b02a4a5..69783c3a99447 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -439,7 +439,7 @@ private static Range parseIpRangeFromCidr(final XContentParser parser) throws IO /** Enum defining the type of range */ public enum RangeType { - IP("ip_range", BinaryDocValuesRangeQuery.LengthType.FIXED_16) { + IP("ip_range", BinaryRangeUtil.LengthType.FIXED_16) { @Override public Field getRangeField(String name, Range r) { return new InetAddressRange(name, (InetAddress)r.from, (InetAddress)r.to); @@ -507,7 +507,7 @@ public Query dvRangeQuery(String field, QueryType queryType, Object from, Object byte[] encodedFrom = InetAddressPoint.encode((InetAddress) from); byte[] encodedTo = InetAddressPoint.encode((InetAddress) to); - return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.FIXED_16, + return new BinaryDocValuesRangeQuery(field, queryType, BinaryRangeUtil.LengthType.FIXED_16, new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); } @@ -546,7 +546,7 @@ private Query createQuery(String field, Object lower, Object upper, boolean incl } } }, - DATE("date_range", BinaryDocValuesRangeQuery.LengthType.VARIABLE, NumberType.LONG) { + DATE("date_range", BinaryRangeUtil.LengthType.VARIABLE, NumberType.LONG) { @Override public Field getRangeField(String name, Range r) { return new LongRange(name, new long[] {((Number)r.from).longValue()}, new long[] {((Number)r.to).longValue()}); @@ -631,7 +631,7 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu } }, // todo support half_float - FLOAT("float_range", BinaryDocValuesRangeQuery.LengthType.FIXED_4, NumberType.FLOAT) { + FLOAT("float_range", BinaryRangeUtil.LengthType.FIXED_4, NumberType.FLOAT) { @Override public Float minValue() { return Float.NEGATIVE_INFINITY; @@ -671,7 +671,7 @@ public Query dvRangeQuery(String field, QueryType queryType, Object from, Object byte[] encodedFrom = BinaryRangeUtil.encodeFloat((Float) from); byte[] encodedTo = BinaryRangeUtil.encodeFloat((Float) to); - return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.FIXED_4, + return new BinaryDocValuesRangeQuery(field, queryType, BinaryRangeUtil.LengthType.FIXED_4, new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); } @@ -695,7 +695,7 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu (f, t) -> FloatRange.newIntersectsQuery(field, new float[] { f }, new float[] { t }), RangeType.FLOAT); } }, - DOUBLE("double_range", BinaryDocValuesRangeQuery.LengthType.FIXED_8, NumberType.DOUBLE) { + DOUBLE("double_range", BinaryRangeUtil.LengthType.FIXED_8, NumberType.DOUBLE) { @Override public Double minValue() { return Double.NEGATIVE_INFINITY; @@ -735,7 +735,7 @@ public Query dvRangeQuery(String field, QueryType queryType, Object from, Object byte[] encodedFrom = BinaryRangeUtil.encodeDouble((Double) from); byte[] encodedTo = BinaryRangeUtil.encodeDouble((Double) to); - return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.FIXED_8, + return new BinaryDocValuesRangeQuery(field, queryType, BinaryRangeUtil.LengthType.FIXED_8, new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); } @@ -762,7 +762,7 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu }, // todo add BYTE support // todo add SHORT support - INTEGER("integer_range", BinaryDocValuesRangeQuery.LengthType.VARIABLE, NumberType.INTEGER) { + INTEGER("integer_range", BinaryRangeUtil.LengthType.VARIABLE, NumberType.INTEGER) { @Override public Integer minValue() { return Integer.MIN_VALUE; @@ -815,7 +815,7 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu (f, t) -> IntRange.newIntersectsQuery(field, new int[] { f }, new int[] { t }), RangeType.INTEGER); } }, - LONG("long_range", BinaryDocValuesRangeQuery.LengthType.VARIABLE, NumberType.LONG) { + LONG("long_range", BinaryRangeUtil.LengthType.VARIABLE, NumberType.LONG) { @Override public Long minValue() { return Long.MIN_VALUE; @@ -855,7 +855,7 @@ public Query dvRangeQuery(String field, QueryType queryType, Object from, Object byte[] encodedFrom = BinaryRangeUtil.encodeLong(((Number) from).longValue()); byte[] encodedTo = BinaryRangeUtil.encodeLong(((Number) to).longValue()); - return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.VARIABLE, + return new BinaryDocValuesRangeQuery(field, queryType, BinaryRangeUtil.LengthType.VARIABLE, new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); } @@ -881,13 +881,13 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu } }; - RangeType(String name, BinaryDocValuesRangeQuery.LengthType lengthType) { + RangeType(String name, BinaryRangeUtil.LengthType lengthType) { this.name = name; this.numberType = null; this.lengthType = lengthType; } - RangeType(String name, BinaryDocValuesRangeQuery.LengthType lengthType, NumberType type) { + RangeType(String name, BinaryRangeUtil.LengthType lengthType, NumberType type) { this.name = name; this.numberType = type; this.lengthType = lengthType; @@ -1005,7 +1005,7 @@ public abstract Query dvRangeQuery(String field, QueryType queryType, Object fro public final String name; private final NumberType numberType; - public final BinaryDocValuesRangeQuery.LengthType lengthType; + public final BinaryRangeUtil.LengthType lengthType; } /** Class defining a range */ diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index 6fd3fa26b595a..32181d76f253b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.queries.BinaryDocValuesRangeQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.test.ESTestCase; @@ -152,7 +151,7 @@ public void testDecodeLong() { for (long expected : cases) { byte[] encoded = BinaryRangeUtil.encodeLong(expected); int offset = 0; - int length = BinaryDocValuesRangeQuery.LengthType.VARIABLE.readLength(encoded, offset); + int length = BinaryRangeUtil.LengthType.VARIABLE.readLength(encoded, offset); assertEquals(expected, BinaryRangeUtil.decodeLong(encoded, offset, length)); } } From bce5737edf937268b00bc0b78a94ac51103cec83 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Mon, 29 Apr 2019 13:40:39 -0400 Subject: [PATCH 11/13] Update tests based on PR feedback --- .../index/mapper/BinaryRangeUtilTests.java | 82 +++++++++++-------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index 32181d76f253b..5830abb647cea 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -157,48 +157,66 @@ public void testDecodeLong() { } public void testDecodeLongRanges() throws IOException { - long start = randomLong(); - long end = randomLongBetween(start + 1, Long.MAX_VALUE); - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.LONG, start, end, true, true); - List decoded = BinaryRangeUtil.decodeLongRanges(BinaryRangeUtil.encodeLongRanges(singleton(expected))); - assertEquals(1, decoded.size()); - RangeFieldMapper.Range actual = decoded.get(0); - assertEquals(expected, actual); + int iters = randomIntBetween(32, 1024); + for (int i = 0; i < iters; i++) { + long start = randomLong(); + long end = randomLongBetween(start + 1, Long.MAX_VALUE); + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.LONG, start, end, true, true); + List decoded = BinaryRangeUtil.decodeLongRanges(BinaryRangeUtil.encodeLongRanges(singleton(expected))); + assertEquals(1, decoded.size()); + RangeFieldMapper.Range actual = decoded.get(0); + assertEquals(expected, actual); + } } public void testDecodeDoubleRanges() throws IOException { - double start = randomDouble(); - double end = randomDoubleBetween(Math.nextUp(start), Double.MAX_VALUE, false); - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, start, end, true, true); - List decoded = BinaryRangeUtil.decodeDoubleRanges(BinaryRangeUtil.encodeDoubleRanges(singleton(expected))); - assertEquals(1, decoded.size()); - RangeFieldMapper.Range actual = decoded.get(0); - assertEquals(expected, actual); + int iters = randomIntBetween(32, 1024); + for (int i = 0; i < iters; i++) { + double start = randomDouble(); + double end = randomDoubleBetween(Math.nextUp(start), Double.MAX_VALUE, false); + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, start, end, true, true); + List decoded = BinaryRangeUtil.decodeDoubleRanges(BinaryRangeUtil.encodeDoubleRanges(singleton(expected))); + assertEquals(1, decoded.size()); + RangeFieldMapper.Range actual = decoded.get(0); + assertEquals(expected, actual); + } } public void testDecodeFloatRanges() throws IOException { - float start = randomFloat(); - // for some reason, ESTestCase doesn't provide randomFloatBetween - float end = randomFloat(); - if (start > end) { - float temp = start; - start = end; - end = temp; + int iters = randomIntBetween(32, 1024); + for (int i = 0; i < iters; i++) { + float start = randomFloat(); + // for some reason, ESTestCase doesn't provide randomFloatBetween + float end = randomFloat(); + if (start > end) { + float temp = start; + start = end; + end = temp; + } + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.FLOAT, start, end, true, true); + List decoded = BinaryRangeUtil.decodeFloatRanges(BinaryRangeUtil.encodeFloatRanges(singleton(expected))); + assertEquals(1, decoded.size()); + RangeFieldMapper.Range actual = decoded.get(0); + assertEquals(expected, actual); } - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.FLOAT, start, end, true, true); - List decoded = BinaryRangeUtil.decodeFloatRanges(BinaryRangeUtil.encodeFloatRanges(singleton(expected))); - assertEquals(1, decoded.size()); - RangeFieldMapper.Range actual = decoded.get(0); - assertEquals(expected, actual); } public void testDecodeIPRanges() throws IOException { - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.IP, InetAddresses.forString("192.168.0.1"), - InetAddresses.forString("192.168.0.100"), true, true); - List decoded = BinaryRangeUtil.decodeIPRanges(BinaryRangeUtil.encodeIPRanges(singleton(expected))); - assertEquals(1, decoded.size()); - RangeFieldMapper.Range actual = decoded.get(0); - assertEquals(expected, actual); + RangeFieldMapper.Range[] cases = { + createIPRange("192.168.0.1", "192.168.0.100"), + createIPRange("::ffff:c0a8:107", "2001:db8::") + }; + for (RangeFieldMapper.Range expected : cases) { + List decoded = BinaryRangeUtil.decodeIPRanges(BinaryRangeUtil.encodeIPRanges(singleton(expected))); + assertEquals(1, decoded.size()); + RangeFieldMapper.Range actual = decoded.get(0); + assertEquals(expected, actual); + } + } + + private RangeFieldMapper.Range createIPRange(String start, String end) { + return new RangeFieldMapper.Range(RangeFieldMapper.RangeType.IP, InetAddresses.forString(start), InetAddresses.forString(end), + true, true); } private static int normalize(int cmp) { From 08305d53d3670cca75e017d98c7e9fe0a302ba2c Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Mon, 29 Apr 2019 13:53:55 -0400 Subject: [PATCH 12/13] Refactor RangeType to a top level class --- .../percolator/PercolatorFieldMapper.java | 2 +- .../queries/BinaryDocValuesRangeQuery.java | 6 +- .../index/mapper/BinaryRangeUtil.java | 52 +- .../index/mapper/RangeFieldMapper.java | 589 ---------------- .../elasticsearch/index/mapper/RangeType.java | 660 ++++++++++++++++++ .../elasticsearch/indices/IndicesModule.java | 3 +- ...ndomBinaryDocValuesRangeQueryTestCase.java | 3 +- .../BinaryDocValuesRangeQueryTests.java | 5 +- ...eRandomBinaryDocValuesRangeQueryTests.java | 6 +- ...tRandomBinaryDocValuesRangeQueryTests.java | 6 +- ...sRandomBinaryDocValuesRangeQueryTests.java | 6 +- ...rRandomBinaryDocValuesRangeQueryTests.java | 6 +- ...gRandomBinaryDocValuesRangeQueryTests.java | 6 +- .../index/mapper/BinaryRangeUtilTests.java | 10 +- .../index/mapper/RangeFieldMapperTests.java | 2 +- ...angeFieldQueryStringQueryBuilderTests.java | 12 +- .../index/mapper/RangeFieldTypeTests.java | 7 +- 17 files changed, 708 insertions(+), 673 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/RangeType.java diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 6ac073ef90a02..40a9e995ad24d 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -68,7 +68,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.RangeFieldMapper; -import org.elasticsearch.index.mapper.RangeFieldMapper.RangeType; +import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; diff --git a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java index 94201a72aae00..d12ecacbdc7d3 100644 --- a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -32,7 +32,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.mapper.BinaryRangeUtil; +import org.elasticsearch.index.mapper.RangeType; import java.io.IOException; import java.util.Objects; @@ -41,13 +41,13 @@ public final class BinaryDocValuesRangeQuery extends Query { private final String fieldName; private final QueryType queryType; - private final BinaryRangeUtil.LengthType lengthType; + private final RangeType.LengthType lengthType; private final BytesRef from; private final BytesRef to; private final Object originalFrom; private final Object originalTo; - public BinaryDocValuesRangeQuery(String fieldName, QueryType queryType, BinaryRangeUtil.LengthType lengthType, + public BinaryDocValuesRangeQuery(String fieldName, QueryType queryType, RangeType.LengthType lengthType, BytesRef from, BytesRef to, Object originalFrom, Object originalTo) { this.fieldName = fieldName; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index b387d69b3cc3e..42157688dd7ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -34,7 +34,7 @@ import java.util.List; import java.util.Set; -public enum BinaryRangeUtil { +enum BinaryRangeUtil { ; @@ -55,7 +55,7 @@ static BytesRef encodeIPRanges(Set ranges) throws IOExce } static List decodeIPRanges(BytesRef encodedRanges) { - return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.IP, BinaryRangeUtil::decodeIP); + return decodeRanges(encodedRanges, RangeType.IP, BinaryRangeUtil::decodeIP); } private static InetAddress decodeIP(byte[] bytes, int offset, int length) { @@ -83,7 +83,7 @@ static BytesRef encodeLongRanges(Set ranges) throws IOEx } static List decodeLongRanges(BytesRef encodedRanges) { - return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.LONG, + return decodeRanges(encodedRanges, RangeType.LONG, BinaryRangeUtil::decodeLong); } @@ -106,19 +106,19 @@ static BytesRef encodeDoubleRanges(Set ranges) throws IO } static List decodeDoubleRanges(BytesRef encodedRanges) { - return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.DOUBLE, + return decodeRanges(encodedRanges, RangeType.DOUBLE, BinaryRangeUtil::decodeDouble); } static List decodeFloatRanges(BytesRef encodedRanges) { - return decodeRanges(encodedRanges, RangeFieldMapper.RangeType.FLOAT, + return decodeRanges(encodedRanges, RangeType.FLOAT, BinaryRangeUtil::decodeFloat); } - static List decodeRanges(BytesRef encodedRanges, RangeFieldMapper.RangeType rangeType, + static List decodeRanges(BytesRef encodedRanges, RangeType rangeType, TriFunction decodeBytes) { - LengthType lengthType = rangeType.lengthType; + RangeType.LengthType lengthType = rangeType.lengthType; ByteArrayDataInput in = new ByteArrayDataInput(); in.reset(encodedRanges.bytes, encodedRanges.offset, encodedRanges.length); int numRanges = in.readVInt(); @@ -257,42 +257,4 @@ private static byte[] encode(long l, int sign) { return encoded; } - public enum LengthType { - FIXED_4 { - @Override - public int readLength(byte[] bytes, int offset) { - return 4; - } - }, - FIXED_8 { - @Override - public int readLength(byte[] bytes, int offset) { - return 8; - } - }, - FIXED_16 { - @Override - public int readLength(byte[] bytes, int offset) { - return 16; - } - }, - VARIABLE { - @Override - public int readLength(byte[] bytes, int offset) { - // the first bit encodes the sign and the next 4 bits encode the number - // of additional bytes - int token = Byte.toUnsignedInt(bytes[offset]); - int length = (token >>> 3) & 0x0f; - if ((token & 0x80) == 0) { - length = 0x0f - length; - } - return 1 + length; - } - }; - - /** - * Return the length of the value that starts at {@code offset} in {@code bytes}. - */ - public abstract int readLength(byte[] bytes, int offset); - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 69783c3a99447..e6a42e59353b0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -19,29 +19,16 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.DoubleRange; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FloatRange; -import org.apache.lucene.document.InetAddressPoint; -import org.apache.lucene.document.InetAddressRange; -import org.apache.lucene.document.IntRange; -import org.apache.lucene.document.LongRange; -import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.BinaryDocValuesRangeQuery; -import org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.geo.ShapeRelation; @@ -54,16 +41,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -71,7 +54,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.BiFunction; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD; @@ -437,577 +419,6 @@ private static Range parseIpRangeFromCidr(final XContentParser parser) throws IO } } - /** Enum defining the type of range */ - public enum RangeType { - IP("ip_range", BinaryRangeUtil.LengthType.FIXED_16) { - @Override - public Field getRangeField(String name, Range r) { - return new InetAddressRange(name, (InetAddress)r.from, (InetAddress)r.to); - } - @Override - public InetAddress parseFrom(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) - throws IOException { - InetAddress address = InetAddresses.forString(parser.text()); - return included ? address : nextUp(address); - } - @Override - public InetAddress parseTo(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) - throws IOException { - InetAddress address = InetAddresses.forString(parser.text()); - return included ? address : nextDown(address); - } - @Override - public InetAddress parse(Object value, boolean coerce) { - if (value instanceof InetAddress) { - return (InetAddress) value; - } else { - if (value instanceof BytesRef) { - value = ((BytesRef) value).utf8ToString(); - } - return InetAddresses.forString(value.toString()); - } - } - @Override - public InetAddress minValue() { - return InetAddressPoint.MIN_VALUE; - } - @Override - public InetAddress maxValue() { - return InetAddressPoint.MAX_VALUE; - } - @Override - public InetAddress nextUp(Object value) { - return InetAddressPoint.nextUp((InetAddress)value); - } - @Override - public InetAddress nextDown(Object value) { - return InetAddressPoint.nextDown((InetAddress)value); - } - - @Override - public BytesRef encodeRanges(Set ranges) throws IOException { - return BinaryRangeUtil.encodeIPRanges(ranges); - } - - @Override - public List decodeRanges(BytesRef bytes) { - // TODO: Implement this. - throw new UnsupportedOperationException(); - } - - @Override - public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { - if (includeFrom == false) { - from = nextUp(from); - } - - if (includeTo == false) { - to = nextDown(to); - } - - byte[] encodedFrom = InetAddressPoint.encode((InetAddress) from); - byte[] encodedTo = InetAddressPoint.encode((InetAddress) to); - return new BinaryDocValuesRangeQuery(field, queryType, BinaryRangeUtil.LengthType.FIXED_16, - new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); - } - - @Override - public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, from, to, includeFrom, includeTo, - (f, t) -> InetAddressRange.newWithinQuery(field, f, t)); - } - @Override - public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, from, to, includeFrom, includeTo, - (f, t) -> InetAddressRange.newContainsQuery(field, f, t )); - } - @Override - public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, from, to, includeFrom, includeTo, - (f, t) -> InetAddressRange.newIntersectsQuery(field, f ,t )); - } - - private Query createQuery(String field, Object lower, Object upper, boolean includeLower, boolean includeUpper, - BiFunction querySupplier) { - byte[] lowerBytes = InetAddressPoint.encode((InetAddress) lower); - byte[] upperBytes = InetAddressPoint.encode((InetAddress) upper); - if (Arrays.compareUnsigned(lowerBytes, 0, lowerBytes.length, upperBytes, 0, upperBytes.length) > 0) { - throw new IllegalArgumentException( - "Range query `from` value (" + lower + ") is greater than `to` value (" + upper + ")"); - } - InetAddress correctedFrom = includeLower ? (InetAddress) lower : nextUp(lower); - InetAddress correctedTo = includeUpper ? (InetAddress) upper : nextDown(upper);; - lowerBytes = InetAddressPoint.encode(correctedFrom); - upperBytes = InetAddressPoint.encode(correctedTo); - if (Arrays.compareUnsigned(lowerBytes, 0, lowerBytes.length, upperBytes, 0, upperBytes.length) > 0) { - return new MatchNoDocsQuery("float range didn't intersect anything"); - } else { - return querySupplier.apply(correctedFrom, correctedTo); - } - } - }, - DATE("date_range", BinaryRangeUtil.LengthType.VARIABLE, NumberType.LONG) { - @Override - public Field getRangeField(String name, Range r) { - return new LongRange(name, new long[] {((Number)r.from).longValue()}, new long[] {((Number)r.to).longValue()}); - } - private Number parse(DateMathParser dateMathParser, String dateStr) { - return dateMathParser.parse(dateStr, () -> {throw new IllegalArgumentException("now is not used at indexing time");}) - .toEpochMilli(); - } - @Override - public Number parseFrom(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) - throws IOException { - Number value = parse(fieldType.dateMathParser, parser.text()); - return included ? value : nextUp(value); - } - @Override - public Number parseTo(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) - throws IOException{ - Number value = parse(fieldType.dateMathParser, parser.text()); - return included ? value : nextDown(value); - } - @Override - public Long minValue() { - return Long.MIN_VALUE; - } - @Override - public Long maxValue() { - return Long.MAX_VALUE; - } - @Override - public Long nextUp(Object value) { - return (long) LONG.nextUp(value); - } - @Override - public Long nextDown(Object value) { - return (long) LONG.nextDown(value); - } - - @Override - public BytesRef encodeRanges(Set ranges) throws IOException { - return LONG.encodeRanges(ranges); - } - - @Override - public List decodeRanges(BytesRef bytes) { - return LONG.decodeRanges(bytes); - } - - @Override - public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { - return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); - } - - @Override - public Query rangeQuery(String field, boolean hasDocValues, Object lowerTerm, Object upperTerm, boolean includeLower, - boolean includeUpper, ShapeRelation relation, @Nullable ZoneId timeZone, - @Nullable DateMathParser parser, QueryShardContext context) { - ZoneId zone = (timeZone == null) ? ZoneOffset.UTC : timeZone; - - DateMathParser dateMathParser = (parser == null) ? - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser() : parser; - Long low = lowerTerm == null ? Long.MIN_VALUE : - dateMathParser.parse(lowerTerm instanceof BytesRef ? ((BytesRef) lowerTerm).utf8ToString() : lowerTerm.toString(), - context::nowInMillis, false, zone).toEpochMilli(); - Long high = upperTerm == null ? Long.MAX_VALUE : - dateMathParser.parse(upperTerm instanceof BytesRef ? ((BytesRef) upperTerm).utf8ToString() : upperTerm.toString(), - context::nowInMillis, false, zone).toEpochMilli(); - - return super.rangeQuery(field, hasDocValues, low, high, includeLower, includeUpper, relation, zone, - dateMathParser, context); - } - @Override - public Query withinQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { - return LONG.withinQuery(field, from, to, includeLower, includeUpper); - } - @Override - public Query containsQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { - return LONG.containsQuery(field, from, to, includeLower, includeUpper); - } - @Override - public Query intersectsQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { - return LONG.intersectsQuery(field, from, to, includeLower, includeUpper); - } - }, - // todo support half_float - FLOAT("float_range", BinaryRangeUtil.LengthType.FIXED_4, NumberType.FLOAT) { - @Override - public Float minValue() { - return Float.NEGATIVE_INFINITY; - } - @Override - public Float maxValue() { - return Float.POSITIVE_INFINITY; - } - @Override - public Float nextUp(Object value) { - return Math.nextUp(((Number)value).floatValue()); - } - @Override - public Float nextDown(Object value) { - return Math.nextDown(((Number)value).floatValue()); - } - - @Override - public BytesRef encodeRanges(Set ranges) throws IOException { - return BinaryRangeUtil.encodeFloatRanges(ranges); - } - - @Override - public List decodeRanges(BytesRef bytes) { - return BinaryRangeUtil.decodeFloatRanges(bytes); - } - - @Override - public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { - if (includeFrom == false) { - from = nextUp(from); - } - - if (includeTo == false) { - to = nextDown(to); - } - - byte[] encodedFrom = BinaryRangeUtil.encodeFloat((Float) from); - byte[] encodedTo = BinaryRangeUtil.encodeFloat((Float) to); - return new BinaryDocValuesRangeQuery(field, queryType, BinaryRangeUtil.LengthType.FIXED_4, - new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); - } - - @Override - public Field getRangeField(String name, Range r) { - return new FloatRange(name, new float[] {((Number)r.from).floatValue()}, new float[] {((Number)r.to).floatValue()}); - } - @Override - public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Float) from, (Float) to, includeFrom, includeTo, - (f, t) -> FloatRange.newWithinQuery(field, new float[] { f }, new float[] { t }), RangeType.FLOAT); - } - @Override - public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Float) from, (Float) to, includeFrom, includeTo, - (f, t) -> FloatRange.newContainsQuery(field, new float[] { f }, new float[] { t }), RangeType.FLOAT); - } - @Override - public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Float) from, (Float) to, includeFrom, includeTo, - (f, t) -> FloatRange.newIntersectsQuery(field, new float[] { f }, new float[] { t }), RangeType.FLOAT); - } - }, - DOUBLE("double_range", BinaryRangeUtil.LengthType.FIXED_8, NumberType.DOUBLE) { - @Override - public Double minValue() { - return Double.NEGATIVE_INFINITY; - } - @Override - public Double maxValue() { - return Double.POSITIVE_INFINITY; - } - @Override - public Double nextUp(Object value) { - return Math.nextUp(((Number)value).doubleValue()); - } - @Override - public Double nextDown(Object value) { - return Math.nextDown(((Number)value).doubleValue()); - } - - @Override - public BytesRef encodeRanges(Set ranges) throws IOException { - return BinaryRangeUtil.encodeDoubleRanges(ranges); - } - - @Override - public List decodeRanges(BytesRef bytes) { - return BinaryRangeUtil.decodeDoubleRanges(bytes); - } - - @Override - public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { - if (includeFrom == false) { - from = nextUp(from); - } - - if (includeTo == false) { - to = nextDown(to); - } - - byte[] encodedFrom = BinaryRangeUtil.encodeDouble((Double) from); - byte[] encodedTo = BinaryRangeUtil.encodeDouble((Double) to); - return new BinaryDocValuesRangeQuery(field, queryType, BinaryRangeUtil.LengthType.FIXED_8, - new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); - } - - @Override - public Field getRangeField(String name, Range r) { - return new DoubleRange(name, new double[] {((Number)r.from).doubleValue()}, new double[] {((Number)r.to).doubleValue()}); - } - @Override - public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Double) from, (Double) to, includeFrom, includeTo, - (f, t) -> DoubleRange.newWithinQuery(field, new double[] { f }, new double[] { t }), RangeType.DOUBLE); - } - @Override - public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Double) from, (Double) to, includeFrom, includeTo, - (f, t) -> DoubleRange.newContainsQuery(field, new double[] { f }, new double[] { t }), RangeType.DOUBLE); - } - @Override - public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Double) from, (Double) to, includeFrom, includeTo, - (f, t) -> DoubleRange.newIntersectsQuery(field, new double[] { f }, new double[] { t }), RangeType.DOUBLE); - } - - }, - // todo add BYTE support - // todo add SHORT support - INTEGER("integer_range", BinaryRangeUtil.LengthType.VARIABLE, NumberType.INTEGER) { - @Override - public Integer minValue() { - return Integer.MIN_VALUE; - } - @Override - public Integer maxValue() { - return Integer.MAX_VALUE; - } - @Override - public Integer nextUp(Object value) { - return ((Number)value).intValue() + 1; - } - @Override - public Integer nextDown(Object value) { - return ((Number)value).intValue() - 1; - } - - @Override - public BytesRef encodeRanges(Set ranges) throws IOException { - return LONG.encodeRanges(ranges); - } - - @Override - public List decodeRanges(BytesRef bytes) { - return LONG.decodeRanges(bytes); - } - - @Override - public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { - return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); - } - - @Override - public Field getRangeField(String name, Range r) { - return new IntRange(name, new int[] {((Number)r.from).intValue()}, new int[] {((Number)r.to).intValue()}); - } - @Override - public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Integer) from, (Integer) to, includeFrom, includeTo, - (f, t) -> IntRange.newWithinQuery(field, new int[] { f }, new int[] { t }), RangeType.INTEGER); - } - @Override - public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Integer) from, (Integer) to, includeFrom, includeTo, - (f, t) -> IntRange.newContainsQuery(field, new int[] { f }, new int[] { t }), RangeType.INTEGER); - } - @Override - public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Integer) from, (Integer) to, includeFrom, includeTo, - (f, t) -> IntRange.newIntersectsQuery(field, new int[] { f }, new int[] { t }), RangeType.INTEGER); - } - }, - LONG("long_range", BinaryRangeUtil.LengthType.VARIABLE, NumberType.LONG) { - @Override - public Long minValue() { - return Long.MIN_VALUE; - } - @Override - public Long maxValue() { - return Long.MAX_VALUE; - } - @Override - public Long nextUp(Object value) { - return ((Number)value).longValue() + 1; - } - @Override - public Long nextDown(Object value) { - return ((Number)value).longValue() - 1; - } - - @Override - public BytesRef encodeRanges(Set ranges) throws IOException { - return BinaryRangeUtil.encodeLongRanges(ranges); - } - - @Override - public List decodeRanges(BytesRef bytes) { - return BinaryRangeUtil.decodeLongRanges(bytes); - } - - @Override - public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { - if (includeFrom == false) { - from = nextUp(from); - } - - if (includeTo == false) { - to = nextDown(to); - } - - byte[] encodedFrom = BinaryRangeUtil.encodeLong(((Number) from).longValue()); - byte[] encodedTo = BinaryRangeUtil.encodeLong(((Number) to).longValue()); - return new BinaryDocValuesRangeQuery(field, queryType, BinaryRangeUtil.LengthType.VARIABLE, - new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); - } - - @Override - public Field getRangeField(String name, Range r) { - return new LongRange(name, new long[] {((Number)r.from).longValue()}, - new long[] {((Number)r.to).longValue()}); - } - @Override - public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Long) from, (Long) to, includeFrom, includeTo, - (f, t) -> LongRange.newWithinQuery(field, new long[] { f }, new long[] { t }), RangeType.LONG); - } - @Override - public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Long) from, (Long) to, includeFrom, includeTo, - (f, t) -> LongRange.newContainsQuery(field, new long[] { f }, new long[] { t }), RangeType.LONG); - } - @Override - public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { - return createQuery(field, (Long) from, (Long) to, includeFrom, includeTo, - (f, t) -> LongRange.newIntersectsQuery(field, new long[] { f }, new long[] { t }), RangeType.LONG); - } - }; - - RangeType(String name, BinaryRangeUtil.LengthType lengthType) { - this.name = name; - this.numberType = null; - this.lengthType = lengthType; - } - - RangeType(String name, BinaryRangeUtil.LengthType lengthType, NumberType type) { - this.name = name; - this.numberType = type; - this.lengthType = lengthType; - } - - /** Get the associated type name. */ - public final String typeName() { - return name; - } - - /** - * Internal helper to create the actual {@link Query} using the provided supplier function. Before creating the query we check if - * the intervals min > max, in which case an {@link IllegalArgumentException} is raised. The method adapts the interval bounds - * based on whether the edges should be included or excluded. In case where after this correction the interval would be empty - * because min > max, we simply return a {@link MatchNoDocsQuery}. - * This helper handles all {@link Number} cases and dates, the IP range type uses its own logic. - */ - private static > Query createQuery(String field, T from, T to, boolean includeFrom, boolean includeTo, - BiFunction querySupplier, RangeType rangeType) { - if (from.compareTo(to) > 0) { - // wrong argument order, this is an error the user should fix - throw new IllegalArgumentException("Range query `from` value (" + from + ") is greater than `to` value (" + to + ")"); - } - - @SuppressWarnings("unchecked") - T correctedFrom = includeFrom ? from : (T) rangeType.nextUp(from); - @SuppressWarnings("unchecked") - T correctedTo = includeTo ? to : (T) rangeType.nextDown(to); - if (correctedFrom.compareTo(correctedTo) > 0) { - return new MatchNoDocsQuery("range didn't intersect anything"); - } else { - return querySupplier.apply(correctedFrom, correctedTo); - } - } - - public abstract Field getRangeField(String name, Range range); - public List createFields(ParseContext context, String name, Range range, boolean indexed, - boolean docValued, boolean stored) { - assert range != null : "range cannot be null when creating fields"; - List fields = new ArrayList<>(); - if (indexed) { - fields.add(getRangeField(name, range)); - } - if (docValued) { - BinaryRangesDocValuesField field = (BinaryRangesDocValuesField) context.doc().getByKey(name); - if (field == null) { - field = new BinaryRangesDocValuesField(name, range, this); - context.doc().addWithKey(name, field); - } else { - field.add(range); - } - } - if (stored) { - fields.add(new StoredField(name, range.toString())); - } - return fields; - } - /** parses from value. rounds according to included flag */ - public Object parseFrom(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { - Number value = numberType.parse(parser, coerce); - return included ? value : (Number)nextUp(value); - } - /** parses to value. rounds according to included flag */ - public Object parseTo(RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { - Number value = numberType.parse(parser, coerce); - return included ? value : (Number)nextDown(value); - } - - public abstract Object minValue(); - public abstract Object maxValue(); - public abstract Object nextUp(Object value); - public abstract Object nextDown(Object value); - public abstract Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); - public abstract Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); - public abstract Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); - public Object parse(Object value, boolean coerce) { - return numberType.parse(value, coerce); - } - public Query rangeQuery(String field, boolean hasDocValues, Object from, Object to, boolean includeFrom, boolean includeTo, - ShapeRelation relation, @Nullable ZoneId timeZone, @Nullable DateMathParser dateMathParser, - QueryShardContext context) { - Object lower = from == null ? minValue() : parse(from, false); - Object upper = to == null ? maxValue() : parse(to, false); - Query indexQuery; - if (relation == ShapeRelation.WITHIN) { - indexQuery = withinQuery(field, lower, upper, includeFrom, includeTo); - } else if (relation == ShapeRelation.CONTAINS) { - indexQuery = containsQuery(field, lower, upper, includeFrom, includeTo); - } else { - indexQuery = intersectsQuery(field, lower, upper, includeFrom, includeTo); - } - if (hasDocValues) { - final QueryType queryType; - if (relation == ShapeRelation.WITHIN) { - queryType = QueryType.WITHIN; - } else if (relation == ShapeRelation.CONTAINS) { - queryType = QueryType.CONTAINS; - } else { - queryType = QueryType.INTERSECTS; - } - Query dvQuery = dvRangeQuery(field, queryType, lower, upper, includeFrom, includeTo); - return new IndexOrDocValuesQuery(indexQuery, dvQuery); - } else { - return indexQuery; - } - } - - // No need to take into account Range#includeFrom or Range#includeTo, because from and to have already been - // rounded up via parseFrom and parseTo methods. - public abstract BytesRef encodeRanges(Set ranges) throws IOException; - public abstract List decodeRanges(BytesRef bytes); - - public abstract Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, - boolean includeFrom, boolean includeTo); - - public final String name; - private final NumberType numberType; - public final BinaryRangeUtil.LengthType lengthType; - } - /** Class defining a range */ public static class Range { RangeType type; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java new file mode 100644 index 0000000000000..3544d75bccdf9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java @@ -0,0 +1,660 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANYDa + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.DoubleRange; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatRange; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.InetAddressRange; +import org.apache.lucene.document.IntRange; +import org.apache.lucene.document.LongRange; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.BinaryDocValuesRangeQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryShardContext; + +import java.io.IOException; +import java.net.InetAddress; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.function.BiFunction; + +/** Enum defining the type of range */ +public enum RangeType { + IP("ip_range", LengthType.FIXED_16) { + @Override + public Field getRangeField(String name, RangeFieldMapper.Range r) { + return new InetAddressRange(name, (InetAddress)r.from, (InetAddress)r.to); + } + @Override + public InetAddress parseFrom(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) + throws IOException { + InetAddress address = InetAddresses.forString(parser.text()); + return included ? address : nextUp(address); + } + @Override + public InetAddress parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) + throws IOException { + InetAddress address = InetAddresses.forString(parser.text()); + return included ? address : nextDown(address); + } + @Override + public InetAddress parse(Object value, boolean coerce) { + if (value instanceof InetAddress) { + return (InetAddress) value; + } else { + if (value instanceof BytesRef) { + value = ((BytesRef) value).utf8ToString(); + } + return InetAddresses.forString(value.toString()); + } + } + @Override + public InetAddress minValue() { + return InetAddressPoint.MIN_VALUE; + } + @Override + public InetAddress maxValue() { + return InetAddressPoint.MAX_VALUE; + } + @Override + public InetAddress nextUp(Object value) { + return InetAddressPoint.nextUp((InetAddress)value); + } + @Override + public InetAddress nextDown(Object value) { + return InetAddressPoint.nextDown((InetAddress)value); + } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return BinaryRangeUtil.encodeIPRanges(ranges); + } + + @Override + public List decodeRanges(BytesRef bytes) { + // TODO: Implement this. + throw new UnsupportedOperationException(); + } + + @Override + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + if (includeFrom == false) { + from = nextUp(from); + } + + if (includeTo == false) { + to = nextDown(to); + } + + byte[] encodedFrom = InetAddressPoint.encode((InetAddress) from); + byte[] encodedTo = InetAddressPoint.encode((InetAddress) to); + return new BinaryDocValuesRangeQuery(field, queryType, LengthType.FIXED_16, + new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); + } + + @Override + public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, from, to, includeFrom, includeTo, + (f, t) -> InetAddressRange.newWithinQuery(field, f, t)); + } + @Override + public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, from, to, includeFrom, includeTo, + (f, t) -> InetAddressRange.newContainsQuery(field, f, t )); + } + @Override + public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, from, to, includeFrom, includeTo, + (f, t) -> InetAddressRange.newIntersectsQuery(field, f ,t )); + } + + private Query createQuery(String field, Object lower, Object upper, boolean includeLower, boolean includeUpper, + BiFunction querySupplier) { + byte[] lowerBytes = InetAddressPoint.encode((InetAddress) lower); + byte[] upperBytes = InetAddressPoint.encode((InetAddress) upper); + if (Arrays.compareUnsigned(lowerBytes, 0, lowerBytes.length, upperBytes, 0, upperBytes.length) > 0) { + throw new IllegalArgumentException( + "Range query `from` value (" + lower + ") is greater than `to` value (" + upper + ")"); + } + InetAddress correctedFrom = includeLower ? (InetAddress) lower : nextUp(lower); + InetAddress correctedTo = includeUpper ? (InetAddress) upper : nextDown(upper);; + lowerBytes = InetAddressPoint.encode(correctedFrom); + upperBytes = InetAddressPoint.encode(correctedTo); + if (Arrays.compareUnsigned(lowerBytes, 0, lowerBytes.length, upperBytes, 0, upperBytes.length) > 0) { + return new MatchNoDocsQuery("float range didn't intersect anything"); + } else { + return querySupplier.apply(correctedFrom, correctedTo); + } + } + }, + DATE("date_range", LengthType.VARIABLE, NumberFieldMapper.NumberType.LONG) { + @Override + public Field getRangeField(String name, RangeFieldMapper.Range r) { + return new LongRange(name, new long[] {((Number)r.from).longValue()}, new long[] {((Number)r.to).longValue()}); + } + private Number parse(DateMathParser dateMathParser, String dateStr) { + return dateMathParser.parse(dateStr, () -> {throw new IllegalArgumentException("now is not used at indexing time");}) + .toEpochMilli(); + } + @Override + public Number parseFrom(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) + throws IOException { + Number value = parse(fieldType.dateMathParser, parser.text()); + return included ? value : nextUp(value); + } + @Override + public Number parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) + throws IOException{ + Number value = parse(fieldType.dateMathParser, parser.text()); + return included ? value : nextDown(value); + } + @Override + public Long minValue() { + return Long.MIN_VALUE; + } + @Override + public Long maxValue() { + return Long.MAX_VALUE; + } + @Override + public Long nextUp(Object value) { + return (long) LONG.nextUp(value); + } + @Override + public Long nextDown(Object value) { + return (long) LONG.nextDown(value); + } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return LONG.encodeRanges(ranges); + } + + @Override + public List decodeRanges(BytesRef bytes) { + return LONG.decodeRanges(bytes); + } + + @Override + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); + } + + @Override + public Query rangeQuery(String field, boolean hasDocValues, Object lowerTerm, Object upperTerm, boolean includeLower, + boolean includeUpper, ShapeRelation relation, @Nullable ZoneId timeZone, + @Nullable DateMathParser parser, QueryShardContext context) { + ZoneId zone = (timeZone == null) ? ZoneOffset.UTC : timeZone; + + DateMathParser dateMathParser = (parser == null) ? + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser() : parser; + Long low = lowerTerm == null ? Long.MIN_VALUE : + dateMathParser.parse(lowerTerm instanceof BytesRef ? ((BytesRef) lowerTerm).utf8ToString() : lowerTerm.toString(), + context::nowInMillis, false, zone).toEpochMilli(); + Long high = upperTerm == null ? Long.MAX_VALUE : + dateMathParser.parse(upperTerm instanceof BytesRef ? ((BytesRef) upperTerm).utf8ToString() : upperTerm.toString(), + context::nowInMillis, false, zone).toEpochMilli(); + + return super.rangeQuery(field, hasDocValues, low, high, includeLower, includeUpper, relation, zone, + dateMathParser, context); + } + @Override + public Query withinQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { + return LONG.withinQuery(field, from, to, includeLower, includeUpper); + } + @Override + public Query containsQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { + return LONG.containsQuery(field, from, to, includeLower, includeUpper); + } + @Override + public Query intersectsQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { + return LONG.intersectsQuery(field, from, to, includeLower, includeUpper); + } + }, + // todo support half_float + FLOAT("float_range", LengthType.FIXED_4, NumberFieldMapper.NumberType.FLOAT) { + @Override + public Float minValue() { + return Float.NEGATIVE_INFINITY; + } + @Override + public Float maxValue() { + return Float.POSITIVE_INFINITY; + } + @Override + public Float nextUp(Object value) { + return Math.nextUp(((Number)value).floatValue()); + } + @Override + public Float nextDown(Object value) { + return Math.nextDown(((Number)value).floatValue()); + } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return BinaryRangeUtil.encodeFloatRanges(ranges); + } + + @Override + public List decodeRanges(BytesRef bytes) { + return BinaryRangeUtil.decodeFloatRanges(bytes); + } + + @Override + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + if (includeFrom == false) { + from = nextUp(from); + } + + if (includeTo == false) { + to = nextDown(to); + } + + byte[] encodedFrom = BinaryRangeUtil.encodeFloat((Float) from); + byte[] encodedTo = BinaryRangeUtil.encodeFloat((Float) to); + return new BinaryDocValuesRangeQuery(field, queryType, LengthType.FIXED_4, + new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); + } + + @Override + public Field getRangeField(String name, RangeFieldMapper.Range r) { + return new FloatRange(name, new float[] {((Number)r.from).floatValue()}, new float[] {((Number)r.to).floatValue()}); + } + @Override + public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Float) from, (Float) to, includeFrom, includeTo, + (f, t) -> FloatRange.newWithinQuery(field, new float[] { f }, new float[] { t }), RangeType.FLOAT); + } + @Override + public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Float) from, (Float) to, includeFrom, includeTo, + (f, t) -> FloatRange.newContainsQuery(field, new float[] { f }, new float[] { t }), RangeType.FLOAT); + } + @Override + public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Float) from, (Float) to, includeFrom, includeTo, + (f, t) -> FloatRange.newIntersectsQuery(field, new float[] { f }, new float[] { t }), RangeType.FLOAT); + } + }, + DOUBLE("double_range", LengthType.FIXED_8, NumberFieldMapper.NumberType.DOUBLE) { + @Override + public Double minValue() { + return Double.NEGATIVE_INFINITY; + } + @Override + public Double maxValue() { + return Double.POSITIVE_INFINITY; + } + @Override + public Double nextUp(Object value) { + return Math.nextUp(((Number)value).doubleValue()); + } + @Override + public Double nextDown(Object value) { + return Math.nextDown(((Number)value).doubleValue()); + } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return BinaryRangeUtil.encodeDoubleRanges(ranges); + } + + @Override + public List decodeRanges(BytesRef bytes) { + return BinaryRangeUtil.decodeDoubleRanges(bytes); + } + + @Override + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + if (includeFrom == false) { + from = nextUp(from); + } + + if (includeTo == false) { + to = nextDown(to); + } + + byte[] encodedFrom = BinaryRangeUtil.encodeDouble((Double) from); + byte[] encodedTo = BinaryRangeUtil.encodeDouble((Double) to); + return new BinaryDocValuesRangeQuery(field, queryType, LengthType.FIXED_8, + new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); + } + + @Override + public Field getRangeField(String name, RangeFieldMapper.Range r) { + return new DoubleRange(name, new double[] {((Number)r.from).doubleValue()}, new double[] {((Number)r.to).doubleValue()}); + } + @Override + public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Double) from, (Double) to, includeFrom, includeTo, + (f, t) -> DoubleRange.newWithinQuery(field, new double[] { f }, new double[] { t }), RangeType.DOUBLE); + } + @Override + public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Double) from, (Double) to, includeFrom, includeTo, + (f, t) -> DoubleRange.newContainsQuery(field, new double[] { f }, new double[] { t }), RangeType.DOUBLE); + } + @Override + public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Double) from, (Double) to, includeFrom, includeTo, + (f, t) -> DoubleRange.newIntersectsQuery(field, new double[] { f }, new double[] { t }), RangeType.DOUBLE); + } + + }, + // todo add BYTE support + // todo add SHORT support + INTEGER("integer_range", LengthType.VARIABLE, NumberFieldMapper.NumberType.INTEGER) { + @Override + public Integer minValue() { + return Integer.MIN_VALUE; + } + @Override + public Integer maxValue() { + return Integer.MAX_VALUE; + } + @Override + public Integer nextUp(Object value) { + return ((Number)value).intValue() + 1; + } + @Override + public Integer nextDown(Object value) { + return ((Number)value).intValue() - 1; + } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return LONG.encodeRanges(ranges); + } + + @Override + public List decodeRanges(BytesRef bytes) { + return LONG.decodeRanges(bytes); + } + + @Override + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); + } + + @Override + public Field getRangeField(String name, RangeFieldMapper.Range r) { + return new IntRange(name, new int[] {((Number)r.from).intValue()}, new int[] {((Number)r.to).intValue()}); + } + @Override + public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Integer) from, (Integer) to, includeFrom, includeTo, + (f, t) -> IntRange.newWithinQuery(field, new int[] { f }, new int[] { t }), RangeType.INTEGER); + } + @Override + public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Integer) from, (Integer) to, includeFrom, includeTo, + (f, t) -> IntRange.newContainsQuery(field, new int[] { f }, new int[] { t }), RangeType.INTEGER); + } + @Override + public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Integer) from, (Integer) to, includeFrom, includeTo, + (f, t) -> IntRange.newIntersectsQuery(field, new int[] { f }, new int[] { t }), RangeType.INTEGER); + } + }, + LONG("long_range", LengthType.VARIABLE, NumberFieldMapper.NumberType.LONG) { + @Override + public Long minValue() { + return Long.MIN_VALUE; + } + @Override + public Long maxValue() { + return Long.MAX_VALUE; + } + @Override + public Long nextUp(Object value) { + return ((Number)value).longValue() + 1; + } + @Override + public Long nextDown(Object value) { + return ((Number)value).longValue() - 1; + } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return BinaryRangeUtil.encodeLongRanges(ranges); + } + + @Override + public List decodeRanges(BytesRef bytes) { + return BinaryRangeUtil.decodeLongRanges(bytes); + } + + @Override + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + if (includeFrom == false) { + from = nextUp(from); + } + + if (includeTo == false) { + to = nextDown(to); + } + + byte[] encodedFrom = BinaryRangeUtil.encodeLong(((Number) from).longValue()); + byte[] encodedTo = BinaryRangeUtil.encodeLong(((Number) to).longValue()); + return new BinaryDocValuesRangeQuery(field, queryType, LengthType.VARIABLE, + new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); + } + + @Override + public Field getRangeField(String name, RangeFieldMapper.Range r) { + return new LongRange(name, new long[] {((Number)r.from).longValue()}, + new long[] {((Number)r.to).longValue()}); + } + @Override + public Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Long) from, (Long) to, includeFrom, includeTo, + (f, t) -> LongRange.newWithinQuery(field, new long[] { f }, new long[] { t }), RangeType.LONG); + } + @Override + public Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Long) from, (Long) to, includeFrom, includeTo, + (f, t) -> LongRange.newContainsQuery(field, new long[] { f }, new long[] { t }), RangeType.LONG); + } + @Override + public Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo) { + return createQuery(field, (Long) from, (Long) to, includeFrom, includeTo, + (f, t) -> LongRange.newIntersectsQuery(field, new long[] { f }, new long[] { t }), RangeType.LONG); + } + }; + + RangeType(String name, LengthType lengthType) { + this.name = name; + this.numberType = null; + this.lengthType = lengthType; + } + + RangeType(String name, LengthType lengthType, NumberFieldMapper.NumberType type) { + this.name = name; + this.numberType = type; + this.lengthType = lengthType; + } + + /** Get the associated type name. */ + public final String typeName() { + return name; + } + + /** + * Internal helper to create the actual {@link Query} using the provided supplier function. Before creating the query we check if + * the intervals min > max, in which case an {@link IllegalArgumentException} is raised. The method adapts the interval bounds + * based on whether the edges should be included or excluded. In case where after this correction the interval would be empty + * because min > max, we simply return a {@link MatchNoDocsQuery}. + * This helper handles all {@link Number} cases and dates, the IP range type uses its own logic. + */ + private static > Query createQuery(String field, T from, T to, boolean includeFrom, boolean includeTo, + BiFunction querySupplier, RangeType rangeType) { + if (from.compareTo(to) > 0) { + // wrong argument order, this is an error the user should fix + throw new IllegalArgumentException("Range query `from` value (" + from + ") is greater than `to` value (" + to + ")"); + } + + @SuppressWarnings("unchecked") + T correctedFrom = includeFrom ? from : (T) rangeType.nextUp(from); + @SuppressWarnings("unchecked") + T correctedTo = includeTo ? to : (T) rangeType.nextDown(to); + if (correctedFrom.compareTo(correctedTo) > 0) { + return new MatchNoDocsQuery("range didn't intersect anything"); + } else { + return querySupplier.apply(correctedFrom, correctedTo); + } + } + + public abstract Field getRangeField(String name, RangeFieldMapper.Range range); + public List createFields(ParseContext context, String name, RangeFieldMapper.Range range, boolean indexed, + boolean docValued, boolean stored) { + assert range != null : "range cannot be null when creating fields"; + List fields = new ArrayList<>(); + if (indexed) { + fields.add(getRangeField(name, range)); + } + if (docValued) { + RangeFieldMapper.BinaryRangesDocValuesField field = (RangeFieldMapper.BinaryRangesDocValuesField) context.doc().getByKey(name); + if (field == null) { + field = new RangeFieldMapper.BinaryRangesDocValuesField(name, range, this); + context.doc().addWithKey(name, field); + } else { + field.add(range); + } + } + if (stored) { + fields.add(new StoredField(name, range.toString())); + } + return fields; + } + /** parses from value. rounds according to included flag */ + public Object parseFrom(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { + Number value = numberType.parse(parser, coerce); + return included ? value : (Number)nextUp(value); + } + /** parses to value. rounds according to included flag */ + public Object parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { + Number value = numberType.parse(parser, coerce); + return included ? value : (Number)nextDown(value); + } + + public abstract Object minValue(); + public abstract Object maxValue(); + public abstract Object nextUp(Object value); + public abstract Object nextDown(Object value); + public abstract Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); + public abstract Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); + public abstract Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); + public Object parse(Object value, boolean coerce) { + return numberType.parse(value, coerce); + } + public Query rangeQuery(String field, boolean hasDocValues, Object from, Object to, boolean includeFrom, boolean includeTo, + ShapeRelation relation, @Nullable ZoneId timeZone, @Nullable DateMathParser dateMathParser, + QueryShardContext context) { + Object lower = from == null ? minValue() : parse(from, false); + Object upper = to == null ? maxValue() : parse(to, false); + Query indexQuery; + if (relation == ShapeRelation.WITHIN) { + indexQuery = withinQuery(field, lower, upper, includeFrom, includeTo); + } else if (relation == ShapeRelation.CONTAINS) { + indexQuery = containsQuery(field, lower, upper, includeFrom, includeTo); + } else { + indexQuery = intersectsQuery(field, lower, upper, includeFrom, includeTo); + } + if (hasDocValues) { + final BinaryDocValuesRangeQuery.QueryType queryType; + if (relation == ShapeRelation.WITHIN) { + queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN; + } else if (relation == ShapeRelation.CONTAINS) { + queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS; + } else { + queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS; + } + Query dvQuery = dvRangeQuery(field, queryType, lower, upper, includeFrom, includeTo); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else { + return indexQuery; + } + } + + // No need to take into account Range#includeFrom or Range#includeTo, because from and to have already been + // rounded up via parseFrom and parseTo methods. + public abstract BytesRef encodeRanges(Set ranges) throws IOException; + public abstract List decodeRanges(BytesRef bytes); + + public abstract Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, + boolean includeFrom, boolean includeTo); + + public final String name; + private final NumberFieldMapper.NumberType numberType; + public final LengthType lengthType; + + public enum LengthType { + FIXED_4 { + @Override + public int readLength(byte[] bytes, int offset) { + return 4; + } + }, + FIXED_8 { + @Override + public int readLength(byte[] bytes, int offset) { + return 8; + } + }, + FIXED_16 { + @Override + public int readLength(byte[] bytes, int offset) { + return 16; + } + }, + VARIABLE { + @Override + public int readLength(byte[] bytes, int offset) { + // the first bit encodes the sign and the next 4 bits encode the number + // of additional bytes + int token = Byte.toUnsignedInt(bytes[offset]); + int length = (token >>> 3) & 0x0f; + if ((token & 0x80) == 0) { + length = 0x0f - length; + } + return 1 + length; + } + }; + + /** + * Return the length of the value that starts at {@code offset} in {@code bytes}. + */ + public abstract int readLength(byte[] bytes, int offset); + }} diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index bef05ecda9fd8..1c673e180d199 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -49,6 +49,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -117,7 +118,7 @@ public static Map getMappers(List mappe for (NumberFieldMapper.NumberType type : NumberFieldMapper.NumberType.values()) { mappers.put(type.typeName(), new NumberFieldMapper.TypeParser(type)); } - for (RangeFieldMapper.RangeType type : RangeFieldMapper.RangeType.values()) { + for (RangeType type : RangeType.values()) { mappers.put(type.typeName(), new RangeFieldMapper.TypeParser(type)); } mappers.put(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser()); diff --git a/server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java b/server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java index dc21ed6a2f799..4629cbb143b00 100644 --- a/server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java +++ b/server/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; import java.io.IOException; import java.util.Collections; @@ -84,7 +85,7 @@ protected final int dimension() { protected abstract String fieldName(); - protected abstract RangeFieldMapper.RangeType rangeType(); + protected abstract RangeType rangeType(); protected abstract static class AbstractRange extends Range { diff --git a/server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java index 921d1ed5f1f23..c214aaaf37475 100644 --- a/server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java @@ -27,6 +27,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -41,7 +42,7 @@ public class BinaryDocValuesRangeQueryTests extends ESTestCase { public void testBasics() throws Exception { String fieldName = "long_field"; - RangeFieldMapper.RangeType rangeType = RangeFieldMapper.RangeType.LONG; + RangeType rangeType = RangeType.LONG; try (Directory dir = newDirectory()) { try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) { // intersects (within) @@ -127,7 +128,7 @@ public void testBasics() throws Exception { public void testNoField() throws IOException { String fieldName = "long_field"; - RangeFieldMapper.RangeType rangeType = RangeFieldMapper.RangeType.LONG; + RangeType rangeType = RangeType.LONG; // no field in index try (Directory dir = newDirectory()) { diff --git a/server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java index 984b1d72ef843..61add8be2a9d6 100644 --- a/server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java @@ -18,7 +18,7 @@ */ package org.apache.lucene.queries; -import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; public class DoubleRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { @@ -28,8 +28,8 @@ protected String fieldName() { } @Override - protected RangeFieldMapper.RangeType rangeType() { - return RangeFieldMapper.RangeType.DOUBLE; + protected RangeType rangeType() { + return RangeType.DOUBLE; } @Override diff --git a/server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java index a7f877392cf43..09755f165af94 100644 --- a/server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java @@ -18,7 +18,7 @@ */ package org.apache.lucene.queries; -import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; public class FloatRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { @@ -28,8 +28,8 @@ protected String fieldName() { } @Override - protected RangeFieldMapper.RangeType rangeType() { - return RangeFieldMapper.RangeType.FLOAT; + protected RangeType rangeType() { + return RangeType.FLOAT; } @Override diff --git a/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java index ec468fd8d9b89..b70616f9b1f52 100644 --- a/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java @@ -20,7 +20,7 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.FutureArrays; -import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; import java.net.InetAddress; import java.net.UnknownHostException; @@ -34,8 +34,8 @@ protected String fieldName() { } @Override - protected RangeFieldMapper.RangeType rangeType() { - return RangeFieldMapper.RangeType.IP; + protected RangeType rangeType() { + return RangeType.IP; } @Override diff --git a/server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java index 1d04cdbaaca86..13c9bd5d32602 100644 --- a/server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java @@ -19,7 +19,7 @@ package org.apache.lucene.queries; import org.apache.lucene.util.TestUtil; -import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; public class IntegerRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { @@ -29,8 +29,8 @@ protected String fieldName() { } @Override - protected RangeFieldMapper.RangeType rangeType() { - return RangeFieldMapper.RangeType.INTEGER; + protected RangeType rangeType() { + return RangeType.INTEGER; } @Override diff --git a/server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java index e506c2c269028..6a8428ab9d3cb 100644 --- a/server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java @@ -19,7 +19,7 @@ package org.apache.lucene.queries; import org.apache.lucene.util.TestUtil; -import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; public class LongRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { @@ -29,8 +29,8 @@ protected String fieldName() { } @Override - protected RangeFieldMapper.RangeType rangeType() { - return RangeFieldMapper.RangeType.LONG; + protected RangeType rangeType() { + return RangeType.LONG; } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index 5830abb647cea..541034c2eb063 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -151,7 +151,7 @@ public void testDecodeLong() { for (long expected : cases) { byte[] encoded = BinaryRangeUtil.encodeLong(expected); int offset = 0; - int length = BinaryRangeUtil.LengthType.VARIABLE.readLength(encoded, offset); + int length = RangeType.LengthType.VARIABLE.readLength(encoded, offset); assertEquals(expected, BinaryRangeUtil.decodeLong(encoded, offset, length)); } } @@ -161,7 +161,7 @@ public void testDecodeLongRanges() throws IOException { for (int i = 0; i < iters; i++) { long start = randomLong(); long end = randomLongBetween(start + 1, Long.MAX_VALUE); - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.LONG, start, end, true, true); + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeType.LONG, start, end, true, true); List decoded = BinaryRangeUtil.decodeLongRanges(BinaryRangeUtil.encodeLongRanges(singleton(expected))); assertEquals(1, decoded.size()); RangeFieldMapper.Range actual = decoded.get(0); @@ -174,7 +174,7 @@ public void testDecodeDoubleRanges() throws IOException { for (int i = 0; i < iters; i++) { double start = randomDouble(); double end = randomDoubleBetween(Math.nextUp(start), Double.MAX_VALUE, false); - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.DOUBLE, start, end, true, true); + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeType.DOUBLE, start, end, true, true); List decoded = BinaryRangeUtil.decodeDoubleRanges(BinaryRangeUtil.encodeDoubleRanges(singleton(expected))); assertEquals(1, decoded.size()); RangeFieldMapper.Range actual = decoded.get(0); @@ -193,7 +193,7 @@ public void testDecodeFloatRanges() throws IOException { start = end; end = temp; } - RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeFieldMapper.RangeType.FLOAT, start, end, true, true); + RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeType.FLOAT, start, end, true, true); List decoded = BinaryRangeUtil.decodeFloatRanges(BinaryRangeUtil.encodeFloatRanges(singleton(expected))); assertEquals(1, decoded.size()); RangeFieldMapper.Range actual = decoded.get(0); @@ -215,7 +215,7 @@ public void testDecodeIPRanges() throws IOException { } private RangeFieldMapper.Range createIPRange(String start, String end) { - return new RangeFieldMapper.Range(RangeFieldMapper.RangeType.IP, InetAddresses.forString(start), InetAddresses.forString(end), + return new RangeFieldMapper.Range(RangeType.IP, InetAddresses.forString(start), InetAddresses.forString(end), true, true); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 65dcd396ed740..913a5d65669fc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -416,7 +416,7 @@ public void doTestNoBounds(String type) throws IOException { public void testIllegalArguments() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("field").field("type", RangeFieldMapper.RangeType.INTEGER.name) + .startObject("properties").startObject("field").field("type", RangeType.INTEGER.name) .field("format", DATE_FORMAT).endObject().endObject().endObject().endObject(); ThrowingRunnable runnable = () -> parser.parse("type", new CompressedXContent(Strings.toString(mapping))); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java index 699f85f1b12b1..fb7386446c1b0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java @@ -69,7 +69,7 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws public void testIntegerRangeQuery() throws Exception { Query query = new QueryStringQueryBuilder(INTEGER_RANGE_FIELD_NAME + ":[-450 TO 45000]").toQuery(createShardContext()); Query range = IntRange.newIntersectsQuery(INTEGER_RANGE_FIELD_NAME, new int[]{-450}, new int[]{45000}); - Query dv = RangeFieldMapper.RangeType.INTEGER.dvRangeQuery(INTEGER_RANGE_FIELD_NAME, + Query dv = RangeType.INTEGER.dvRangeQuery(INTEGER_RANGE_FIELD_NAME, BinaryDocValuesRangeQuery.QueryType.INTERSECTS, -450, 45000, true, true); assertEquals(new IndexOrDocValuesQuery(range, dv), query); } @@ -77,7 +77,7 @@ public void testIntegerRangeQuery() throws Exception { public void testLongRangeQuery() throws Exception { Query query = new QueryStringQueryBuilder(LONG_RANGE_FIELD_NAME + ":[-450 TO 45000]").toQuery(createShardContext()); Query range = LongRange.newIntersectsQuery(LONG_RANGE_FIELD_NAME, new long[]{-450}, new long[]{45000}); - Query dv = RangeFieldMapper.RangeType.LONG.dvRangeQuery(LONG_RANGE_FIELD_NAME, + Query dv = RangeType.LONG.dvRangeQuery(LONG_RANGE_FIELD_NAME, BinaryDocValuesRangeQuery.QueryType.INTERSECTS, -450, 45000, true, true); assertEquals(new IndexOrDocValuesQuery(range, dv), query); } @@ -85,7 +85,7 @@ public void testLongRangeQuery() throws Exception { public void testFloatRangeQuery() throws Exception { Query query = new QueryStringQueryBuilder(FLOAT_RANGE_FIELD_NAME + ":[-450 TO 45000]").toQuery(createShardContext()); Query range = FloatRange.newIntersectsQuery(FLOAT_RANGE_FIELD_NAME, new float[]{-450}, new float[]{45000}); - Query dv = RangeFieldMapper.RangeType.FLOAT.dvRangeQuery(FLOAT_RANGE_FIELD_NAME, + Query dv = RangeType.FLOAT.dvRangeQuery(FLOAT_RANGE_FIELD_NAME, BinaryDocValuesRangeQuery.QueryType.INTERSECTS, -450.0f, 45000.0f, true, true); assertEquals(new IndexOrDocValuesQuery(range, dv), query); } @@ -93,7 +93,7 @@ public void testFloatRangeQuery() throws Exception { public void testDoubleRangeQuery() throws Exception { Query query = new QueryStringQueryBuilder(DOUBLE_RANGE_FIELD_NAME + ":[-450 TO 45000]").toQuery(createShardContext()); Query range = DoubleRange.newIntersectsQuery(DOUBLE_RANGE_FIELD_NAME, new double[]{-450}, new double[]{45000}); - Query dv = RangeFieldMapper.RangeType.DOUBLE.dvRangeQuery(DOUBLE_RANGE_FIELD_NAME, + Query dv = RangeType.DOUBLE.dvRangeQuery(DOUBLE_RANGE_FIELD_NAME, BinaryDocValuesRangeQuery.QueryType.INTERSECTS, -450.0, 45000.0, true, true); assertEquals(new IndexOrDocValuesQuery(range, dv), query); } @@ -106,7 +106,7 @@ public void testDateRangeQuery() throws Exception { Query range = LongRange.newIntersectsQuery(DATE_RANGE_FIELD_NAME, new long[]{ parser.parse("2010-01-01", () -> 0).toEpochMilli()}, new long[]{ parser.parse("2018-01-01", () -> 0).toEpochMilli()}); - Query dv = RangeFieldMapper.RangeType.DATE.dvRangeQuery(DATE_RANGE_FIELD_NAME, + Query dv = RangeType.DATE.dvRangeQuery(DATE_RANGE_FIELD_NAME, BinaryDocValuesRangeQuery.QueryType.INTERSECTS, parser.parse("2010-01-01", () -> 0).toEpochMilli(), parser.parse("2018-01-01", () -> 0).toEpochMilli(), true, true); @@ -118,7 +118,7 @@ public void testIPRangeQuery() throws Exception { InetAddress upper = InetAddresses.forString("192.168.0.5"); Query query = new QueryStringQueryBuilder(IP_RANGE_FIELD_NAME + ":[192.168.0.1 TO 192.168.0.5]").toQuery(createShardContext()); Query range = InetAddressRange.newIntersectsQuery(IP_RANGE_FIELD_NAME, lower, upper); - Query dv = RangeFieldMapper.RangeType.IP.dvRangeQuery(IP_RANGE_FIELD_NAME, + Query dv = RangeType.IP.dvRangeQuery(IP_RANGE_FIELD_NAME, BinaryDocValuesRangeQuery.QueryType.INTERSECTS, lower, upper, true, true); assertEquals(new IndexOrDocValuesQuery(range, dv), query); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index a26999fa3a6f5..29d70d413729f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -40,7 +40,6 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.RangeFieldMapper.RangeFieldType; -import org.elasticsearch.index.mapper.RangeFieldMapper.RangeType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.test.IndexSettingsModule; import org.joda.time.DateTime; @@ -434,9 +433,9 @@ private Object nextTo(Object from) throws Exception { } public void testParseIp() { - assertEquals(InetAddresses.forString("::1"), RangeFieldMapper.RangeType.IP.parse(InetAddresses.forString("::1"), randomBoolean())); - assertEquals(InetAddresses.forString("::1"), RangeFieldMapper.RangeType.IP.parse("::1", randomBoolean())); - assertEquals(InetAddresses.forString("::1"), RangeFieldMapper.RangeType.IP.parse(new BytesRef("::1"), randomBoolean())); + assertEquals(InetAddresses.forString("::1"), RangeType.IP.parse(InetAddresses.forString("::1"), randomBoolean())); + assertEquals(InetAddresses.forString("::1"), RangeType.IP.parse("::1", randomBoolean())); + assertEquals(InetAddresses.forString("::1"), RangeType.IP.parse(new BytesRef("::1"), randomBoolean())); } public void testTermQuery() throws Exception { From fa8ff707e354ceb551b8f35c14851841f284deb1 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Mon, 29 Apr 2019 14:45:42 -0400 Subject: [PATCH 13/13] Fix checkstyle issues --- .../elasticsearch/index/mapper/RangeType.java | 27 ++++++++++++------- .../index/mapper/BinaryRangeUtilTests.java | 6 +++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java index 3544d75bccdf9..ac3ec8f750603 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java @@ -109,7 +109,8 @@ public List decodeRanges(BytesRef bytes) { } @Override - public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, + boolean includeTo) { if (includeFrom == false) { from = nextUp(from); } @@ -208,7 +209,8 @@ public List decodeRanges(BytesRef bytes) { } @Override - public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, + boolean includeTo) { return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); } @@ -273,7 +275,8 @@ public List decodeRanges(BytesRef bytes) { } @Override - public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, + boolean includeTo) { if (includeFrom == false) { from = nextUp(from); } @@ -337,7 +340,8 @@ public List decodeRanges(BytesRef bytes) { } @Override - public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, + boolean includeTo) { if (includeFrom == false) { from = nextUp(from); } @@ -404,7 +408,8 @@ public List decodeRanges(BytesRef bytes) { } @Override - public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, + boolean includeTo) { return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); } @@ -457,7 +462,8 @@ public List decodeRanges(BytesRef bytes) { } @Override - public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + public Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.QueryType queryType, Object from, Object to, boolean includeFrom, + boolean includeTo) { if (includeFrom == false) { from = nextUp(from); } @@ -559,12 +565,14 @@ public List createFields(ParseContext context, String name, Rang return fields; } /** parses from value. rounds according to included flag */ - public Object parseFrom(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { + public Object parseFrom(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, + boolean included) throws IOException { Number value = numberType.parse(parser, coerce); return included ? value : (Number)nextUp(value); } /** parses to value. rounds according to included flag */ - public Object parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { + public Object parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, + boolean included) throws IOException { Number value = numberType.parse(parser, coerce); return included ? value : (Number)nextDown(value); } @@ -657,4 +665,5 @@ public int readLength(byte[] bytes, int offset) { * Return the length of the value that starts at {@code offset} in {@code bytes}. */ public abstract int readLength(byte[] bytes, int offset); - }} + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index 541034c2eb063..546cbbb2eaa6b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -175,7 +175,8 @@ public void testDecodeDoubleRanges() throws IOException { double start = randomDouble(); double end = randomDoubleBetween(Math.nextUp(start), Double.MAX_VALUE, false); RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeType.DOUBLE, start, end, true, true); - List decoded = BinaryRangeUtil.decodeDoubleRanges(BinaryRangeUtil.encodeDoubleRanges(singleton(expected))); + List decoded = BinaryRangeUtil.decodeDoubleRanges(BinaryRangeUtil.encodeDoubleRanges( + singleton(expected))); assertEquals(1, decoded.size()); RangeFieldMapper.Range actual = decoded.get(0); assertEquals(expected, actual); @@ -194,7 +195,8 @@ public void testDecodeFloatRanges() throws IOException { end = temp; } RangeFieldMapper.Range expected = new RangeFieldMapper.Range(RangeType.FLOAT, start, end, true, true); - List decoded = BinaryRangeUtil.decodeFloatRanges(BinaryRangeUtil.encodeFloatRanges(singleton(expected))); + List decoded = BinaryRangeUtil.decodeFloatRanges(BinaryRangeUtil.encodeFloatRanges( + singleton(expected))); assertEquals(1, decoded.size()); RangeFieldMapper.Range actual = decoded.get(0); assertEquals(expected, actual);