From 175426a9938ea32d9e6b50c96a181507cc08f56d Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 1 Sep 2017 13:42:50 +0200 Subject: [PATCH] More efficient encoding of range fields. This PR removes the vInt that precedes every value in order to know how long they are. Instead the query takes an enum that tells how to compute the length of values: for fixed-length data (ip addresses, double, float) the length is a constant while longs and integers use a variable-length representation that allows the length to be computed from the encoded values. Also the encoding of ints/longs was made a bit more efficient in order not to waste 3 bits in the header. As a consequence, values between -8 and 7 can now be encoded on 1 byte and values between -2048 and 2047 can now be encoded on 2 bytes or less. Closes #26443 --- .../queries/BinaryDocValuesRangeQuery.java | 71 ++++++-- .../index/mapper/BinaryRangeUtil.java | 153 ++++++++++-------- .../index/mapper/RangeFieldMapper.java | 93 +++++++---- .../index/mapper/BinaryRangeUtilTests.java | 87 ++++++++-- 4 files changed, 273 insertions(+), 131 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java index c8f78ab616d3f..f5d86849e56d1 100644 --- a/core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -37,15 +37,18 @@ public final class BinaryDocValuesRangeQuery extends Query { private final String fieldName; private final QueryType queryType; + private final LengthType lengthType; private final BytesRef from; private final BytesRef to; private final Object originalFrom; private final Object originalTo; - public BinaryDocValuesRangeQuery(String fieldName, QueryType queryType, BytesRef from, BytesRef to, + public BinaryDocValuesRangeQuery(String fieldName, QueryType queryType, LengthType lengthType, + BytesRef from, BytesRef to, Object originalFrom, Object originalTo) { this.fieldName = fieldName; this.queryType = queryType; + this.lengthType = lengthType; this.from = from; this.to = to; this.originalFrom = originalFrom; @@ -66,29 +69,34 @@ public Scorer scorer(LeafReaderContext context) throws IOException { final TwoPhaseIterator iterator = new TwoPhaseIterator(values) { ByteArrayDataInput in = new ByteArrayDataInput(); - BytesRef otherFrom = new BytesRef(16); - BytesRef otherTo = new BytesRef(16); + BytesRef otherFrom = new BytesRef(); + BytesRef otherTo = new BytesRef(); @Override public boolean matches() throws IOException { BytesRef encodedRanges = values.binaryValue(); in.reset(encodedRanges.bytes, encodedRanges.offset, encodedRanges.length); int numRanges = in.readVInt(); + final byte[] bytes = encodedRanges.bytes; + otherFrom.bytes = bytes; + otherTo.bytes = bytes; + int offset = in.getPosition(); for (int i = 0; i < numRanges; i++) { - otherFrom.length = in.readVInt(); - otherFrom.bytes = encodedRanges.bytes; - otherFrom.offset = in.getPosition(); - in.skipBytes(otherFrom.length); + int length = lengthType.readLength(bytes, offset); + otherFrom.offset = offset; + otherFrom.length = length; + offset += length; - otherTo.length = in.readVInt(); - otherTo.bytes = encodedRanges.bytes; - otherTo.offset = in.getPosition(); - in.skipBytes(otherTo.length); + length = lengthType.readLength(bytes, offset); + otherTo.offset = offset; + otherTo.length = length; + offset += length; if (queryType.matches(from, to, otherFrom, otherTo)) { return true; } } + assert offset == encodedRanges.offset + encodedRanges.length; return false; } @@ -114,13 +122,14 @@ public boolean equals(Object o) { BinaryDocValuesRangeQuery that = (BinaryDocValuesRangeQuery) o; return Objects.equals(fieldName, that.fieldName) && queryType == that.queryType && + lengthType == that.lengthType && Objects.equals(from, that.from) && Objects.equals(to, that.to); } @Override public int hashCode() { - return Objects.hash(getClass(), fieldName, queryType, from, to); + return Objects.hash(getClass(), fieldName, queryType, lengthType, from, to); } public enum QueryType { @@ -161,4 +170,42 @@ boolean matches(BytesRef from, BytesRef to, BytesRef otherFrom, BytesRef otherTo } + public enum LengthType { + FIXED_4 { + @Override + int readLength(byte[] bytes, int offset) { + return 4; + } + }, + FIXED_8 { + @Override + int readLength(byte[] bytes, int offset) { + return 8; + } + }, + FIXED_16 { + @Override + int readLength(byte[] bytes, int offset) { + return 16; + } + }, + VARIABLE { + @Override + int readLength(byte[] bytes, int offset) { + // the first bit encodes the sign and the next 4 bits encode the number + // of additional bytes + int token = Byte.toUnsignedInt(bytes[offset]); + int length = (token >>> 3) & 0x0f; + if ((token & 0x80) == 0) { + length = 0x0f - length; + } + return 1 + length; + } + }; + + /** + * Return the length of the value that starts at {@code offset} in {@code bytes}. + */ + abstract int readLength(byte[] bytes, int offset); + } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java index e2b618bc222e7..384ab24a73bf6 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -18,11 +18,14 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.HalfFloatPoint; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; import java.io.IOException; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Set; @@ -32,28 +35,17 @@ enum BinaryRangeUtil { static BytesRef encodeLongRanges(Set ranges) throws IOException { List sortedRanges = new ArrayList<>(ranges); - sortedRanges.sort((r1, r2) -> { - long r1From = ((Number) r1.from).longValue(); - long r2From = ((Number) r2.from).longValue(); - int cmp = Long.compare(r1From, r2From); - if (cmp != 0) { - return cmp; - } else { - long r1To = ((Number) r1.from).longValue(); - long r2To = ((Number) r2.from).longValue(); - return Long.compare(r1To, r2To); - } - }); + Comparator fromComparator = Comparator.comparingLong(range -> ((Number) range.from).longValue()); + Comparator toComparator = Comparator.comparingLong(range -> ((Number) range.to).longValue()); + sortedRanges.sort(fromComparator.thenComparing(toComparator)); - final byte[] encoded = new byte[5 + ((5 + 9) * 2) * sortedRanges.size()]; + final byte[] encoded = new byte[5 + (9 * 2) * sortedRanges.size()]; ByteArrayDataOutput out = new ByteArrayDataOutput(encoded); out.writeVInt(sortedRanges.size()); for (RangeFieldMapper.Range range : sortedRanges) { - byte[] encodedFrom = encode(((Number) range.from).longValue()); - out.writeVInt(encodedFrom.length); + byte[] encodedFrom = encodeLong(((Number) range.from).longValue()); out.writeBytes(encodedFrom, encodedFrom.length); - byte[] encodedTo = encode(((Number) range.to).longValue()); - out.writeVInt(encodedTo.length); + byte[] encodedTo = encodeLong(((Number) range.to).longValue()); out.writeBytes(encodedTo, encodedTo.length); } return new BytesRef(encoded, 0, out.getPosition()); @@ -61,38 +53,59 @@ static BytesRef encodeLongRanges(Set ranges) throws IOEx static BytesRef encodeDoubleRanges(Set ranges) throws IOException { List sortedRanges = new ArrayList<>(ranges); - sortedRanges.sort((r1, r2) -> { - double r1From = ((Number) r1.from).doubleValue(); - double r2From = ((Number) r2.from).doubleValue(); - int cmp = Double.compare(r1From, r2From); - if (cmp != 0) { - return cmp; - } else { - double r1To = ((Number) r1.from).doubleValue(); - double r2To = ((Number) r2.from).doubleValue(); - return Double.compare(r1To, r2To); - } - }); + Comparator fromComparator = Comparator.comparingDouble(range -> ((Number) range.from).doubleValue()); + Comparator toComparator = Comparator.comparingDouble(range -> ((Number) range.to).doubleValue()); + sortedRanges.sort(fromComparator.thenComparing(toComparator)); - final byte[] encoded = new byte[5 + ((5 + 9) * 2) * sortedRanges.size()]; + final byte[] encoded = new byte[5 + (8 * 2) * sortedRanges.size()]; ByteArrayDataOutput out = new ByteArrayDataOutput(encoded); out.writeVInt(sortedRanges.size()); for (RangeFieldMapper.Range range : sortedRanges) { - byte[] encodedFrom = BinaryRangeUtil.encode(((Number) range.from).doubleValue()); - out.writeVInt(encodedFrom.length); + byte[] encodedFrom = encodeDouble(((Number) range.from).doubleValue()); out.writeBytes(encodedFrom, encodedFrom.length); - byte[] encodedTo = BinaryRangeUtil.encode(((Number) range.to).doubleValue()); - out.writeVInt(encodedTo.length); + byte[] encodedTo = encodeDouble(((Number) range.to).doubleValue()); out.writeBytes(encodedTo, encodedTo.length); } return new BytesRef(encoded, 0, out.getPosition()); } + static BytesRef encodeFloatRanges(Set ranges) throws IOException { + List sortedRanges = new ArrayList<>(ranges); + Comparator fromComparator = Comparator.comparingDouble(range -> ((Number) range.from).floatValue()); + Comparator toComparator = Comparator.comparingDouble(range -> ((Number) range.to).floatValue()); + sortedRanges.sort(fromComparator.thenComparing(toComparator)); + + final byte[] encoded = new byte[5 + (4 * 2) * sortedRanges.size()]; + ByteArrayDataOutput out = new ByteArrayDataOutput(encoded); + out.writeVInt(sortedRanges.size()); + for (RangeFieldMapper.Range range : sortedRanges) { + byte[] encodedFrom = encodeFloat(((Number) range.from).floatValue()); + out.writeBytes(encodedFrom, encodedFrom.length); + byte[] encodedTo = encodeFloat(((Number) range.to).floatValue()); + out.writeBytes(encodedTo, encodedTo.length); + } + return new BytesRef(encoded, 0, out.getPosition()); + } + + static byte[] encodeDouble(double number) { + byte[] encoded = new byte[8]; + NumericUtils.longToSortableBytes(NumericUtils.doubleToSortableLong(number), encoded, 0); + return encoded; + } + + static byte[] encodeFloat(float number) { + byte[] encoded = new byte[4]; + NumericUtils.intToSortableBytes(NumericUtils.floatToSortableInt(number), encoded, 0); + return encoded; + } + /** * Encodes the specified number of type long in a variable-length byte format. * The byte format preserves ordering, which means the returned byte array can be used for comparing as is. + * The first bit stores the sign and the 4 subsequent bits encode the number of bytes that are used to + * represent the long value, in addition to the first one. */ - static byte[] encode(long number) { + static byte[] encodeLong(long number) { int sign = 1; // means positive if (number < 0) { number = -1 - number; @@ -101,46 +114,48 @@ static byte[] encode(long number) { return encode(number, sign); } - /** - * Encodes the specified number of type double in a variable-length byte format. - * The byte format preserves ordering, which means the returned byte array can be used for comparing as is. - */ - static byte[] encode(double number) { - long l; - int sign; - if (number < 0.0) { - l = Double.doubleToRawLongBits(-0d - number); - sign = 0; - } else { - l = Double.doubleToRawLongBits(number); - sign = 1; // means positive - } - return encode(l, sign); - } - private static byte[] encode(long l, int sign) { assert l >= 0; - int bits = 64 - Long.numberOfLeadingZeros(l); - int numBytes = (bits + 7) / 8; // between 0 and 8 - byte[] encoded = new byte[1 + numBytes]; - // encode the sign first to make sure positive values compare greater than negative values - // and then the number of bytes, to make sure that large values compare greater than low values - if (sign > 0) { - encoded[0] = (byte) ((sign << 4) | numBytes); - } else { - encoded[0] = (byte) ((sign << 4) | (8 - numBytes)); + // the header is formed of: + // - 1 bit for the sign + // - 4 bits for the number of additional bytes + // - up to 3 bits of the value + // additional bytes are data bytes + + int numBits = 64 - Long.numberOfLeadingZeros(l); + int numAdditionalBytes = (numBits + 7 - 3) / 8; + + byte[] encoded = new byte[1 + numAdditionalBytes]; + + // write data bytes + int i = encoded.length; + while (numBits > 0) { + int index = --i; + assert index > 0 || numBits <= 3; // byte 0 can't encode more than 3 bits + encoded[index] = (byte) l; + l >>>= 8; + numBits -= 8; } - for (int b = 0; b < numBytes; ++b) { - if (sign == 1) { - encoded[encoded.length - 1 - b] = (byte) (l >>> (8 * b)); - } else if (sign == 0) { - encoded[encoded.length - 1 - b] = (byte) (0xFF - ((l >>> (8 * b)) & 0xFF)); - } else { - throw new AssertionError(); + assert Byte.toUnsignedInt(encoded[0]) <= 0x07; + assert encoded.length == 1 || encoded[0] != 0 || Byte.toUnsignedInt(encoded[1]) > 0x07; + + if (sign == 0) { + // reverse the order + for (int j = 0; j < encoded.length; ++j) { + encoded[j] = (byte) ~Byte.toUnsignedInt(encoded[j]); } + // the first byte only uses 3 bits, we need the 5 upper bits for the header + encoded[0] &= 0x07; + } + + // write the header + encoded[0] |= sign << 7; + if (sign > 0) { + encoded[0] |= numAdditionalBytes << 3; + } else { + encoded[0] |= (15 - numAdditionalBytes) << 3; } return encoded; } - } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index ed77e0a97aa1f..c243858cd9155 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -477,12 +477,10 @@ public BytesRef encodeRanges(Set ranges) throws IOException { ByteArrayDataOutput out = new ByteArrayDataOutput(encoded); out.writeVInt(ranges.size()); for (Range range : ranges) { - out.writeVInt(16); InetAddress fromValue = (InetAddress) range.from; byte[] encodedFromValue = InetAddressPoint.encode(fromValue); out.writeBytes(encodedFromValue, 0, encodedFromValue.length); - out.writeVInt(16); InetAddress toValue = (InetAddress) range.to; byte[] encodedToValue = InetAddressPoint.encode(toValue); out.writeBytes(encodedToValue, 0, encodedToValue.length); @@ -491,10 +489,19 @@ public BytesRef encodeRanges(Set ranges) throws IOException { } @Override - BytesRef[] encodeRange(Object from, Object to) { - BytesRef encodedFrom = new BytesRef(InetAddressPoint.encode((InetAddress) from)); - BytesRef encodedTo = new BytesRef(InetAddressPoint.encode((InetAddress) to)); - return new BytesRef[]{encodedFrom, encodedTo}; + public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + if (includeFrom == false) { + from = nextUp(from); + } + + if (includeTo == false) { + to = nextDown(to); + } + + byte[] encodedFrom = InetAddressPoint.encode((InetAddress) from); + byte[] encodedTo = InetAddressPoint.encode((InetAddress) to); + return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.FIXED_16, + new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); } @Override @@ -565,8 +572,8 @@ public BytesRef encodeRanges(Set ranges) throws IOException { } @Override - BytesRef[] encodeRange(Object from, Object to) { - return LONG.encodeRange(from, to); + public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); } @Override @@ -620,12 +627,23 @@ public Float nextDown(Object value) { @Override public BytesRef encodeRanges(Set ranges) throws IOException { - return DOUBLE.encodeRanges(ranges); + return BinaryRangeUtil.encodeFloatRanges(ranges); } @Override - BytesRef[] encodeRange(Object from, Object to) { - return DOUBLE.encodeRange(((Number) from).floatValue(), ((Number) to).floatValue()); + public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + if (includeFrom == false) { + from = nextUp(from); + } + + if (includeTo == false) { + to = nextDown(to); + } + + byte[] encodedFrom = BinaryRangeUtil.encodeFloat((Float) from); + byte[] encodedTo = BinaryRangeUtil.encodeFloat((Float) to); + return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.FIXED_4, + new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); } @Override @@ -675,10 +693,19 @@ public BytesRef encodeRanges(Set ranges) throws IOException { } @Override - BytesRef[] encodeRange(Object from, Object to) { - byte[] fromValue = BinaryRangeUtil.encode(((Number) from).doubleValue()); - byte[] toValue = BinaryRangeUtil.encode(((Number) to).doubleValue()); - return new BytesRef[]{new BytesRef(fromValue), new BytesRef(toValue)}; + public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + if (includeFrom == false) { + from = nextUp(from); + } + + if (includeTo == false) { + to = nextDown(to); + } + + byte[] encodedFrom = BinaryRangeUtil.encodeDouble((Double) from); + byte[] encodedTo = BinaryRangeUtil.encodeDouble((Double) to); + return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.FIXED_8, + new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); } @Override @@ -730,8 +757,8 @@ public BytesRef encodeRanges(Set ranges) throws IOException { } @Override - BytesRef[] encodeRange(Object from, Object to) { - return LONG.encodeRange(from, to); + public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo); } @Override @@ -778,10 +805,19 @@ public BytesRef encodeRanges(Set ranges) throws IOException { } @Override - BytesRef[] encodeRange(Object from, Object to) { - byte[] encodedFrom = BinaryRangeUtil.encode(((Number) from).longValue()); - byte[] encodedTo = BinaryRangeUtil.encode(((Number) to).longValue()); - return new BytesRef[]{new BytesRef(encodedFrom), new BytesRef(encodedTo)}; + public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + if (includeFrom == false) { + from = nextUp(from); + } + + if (includeTo == false) { + to = nextDown(to); + } + + byte[] encodedFrom = BinaryRangeUtil.encodeLong(((Number) from).longValue()); + byte[] encodedTo = BinaryRangeUtil.encodeLong(((Number) to).longValue()); + return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.VARIABLE, + new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to); } @Override @@ -897,19 +933,8 @@ public Query rangeQuery(String field, boolean hasDocValues, Object from, Object // rounded up via parseFrom and parseTo methods. public abstract BytesRef encodeRanges(Set ranges) throws IOException; - public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { - if (includeFrom == false) { - from = nextUp(from); - } - - if (includeTo == false) { - to = nextDown(to); - } - BytesRef[] range = encodeRange(from, to); - return new BinaryDocValuesRangeQuery(field, queryType, range[0], range[1], from, to); - } - - abstract BytesRef[] encodeRange(Object from, Object to); + public abstract Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, + boolean includeFrom, boolean includeTo); public final String name; private final NumberType numberType; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java index 8a4e6945ffc36..20d4af1f0b600 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -24,11 +24,11 @@ public class BinaryRangeUtilTests extends ESTestCase { public void testBasics() { - BytesRef encoded1 = new BytesRef(BinaryRangeUtil.encode(Long.MIN_VALUE)); - BytesRef encoded2 = new BytesRef(BinaryRangeUtil.encode(-1L)); - BytesRef encoded3 = new BytesRef(BinaryRangeUtil.encode(0L)); - BytesRef encoded4 = new BytesRef(BinaryRangeUtil.encode(1L)); - BytesRef encoded5 = new BytesRef(BinaryRangeUtil.encode(Long.MAX_VALUE)); + BytesRef encoded1 = new BytesRef(BinaryRangeUtil.encodeLong(Long.MIN_VALUE)); + BytesRef encoded2 = new BytesRef(BinaryRangeUtil.encodeLong(-1L)); + BytesRef encoded3 = new BytesRef(BinaryRangeUtil.encodeLong(0L)); + BytesRef encoded4 = new BytesRef(BinaryRangeUtil.encodeLong(1L)); + BytesRef encoded5 = new BytesRef(BinaryRangeUtil.encodeLong(Long.MAX_VALUE)); assertTrue(encoded1.compareTo(encoded2) < 0); assertTrue(encoded2.compareTo(encoded1) > 0); @@ -39,11 +39,12 @@ public void testBasics() { assertTrue(encoded4.compareTo(encoded5) < 0); assertTrue(encoded5.compareTo(encoded4) > 0); - encoded1 = new BytesRef(BinaryRangeUtil.encode(Double.NEGATIVE_INFINITY)); - encoded2 = new BytesRef(BinaryRangeUtil.encode(-1D)); - encoded3 = new BytesRef(BinaryRangeUtil.encode(0D)); - encoded4 = new BytesRef(BinaryRangeUtil.encode(1D)); - encoded5 = new BytesRef(BinaryRangeUtil.encode(Double.POSITIVE_INFINITY)); + encoded1 = new BytesRef(BinaryRangeUtil.encodeDouble(Double.NEGATIVE_INFINITY)); + encoded2 = new BytesRef(BinaryRangeUtil.encodeDouble(-1D)); + encoded3 = new BytesRef(BinaryRangeUtil.encodeDouble(-0D)); + encoded4 = new BytesRef(BinaryRangeUtil.encodeDouble(0D)); + encoded5 = new BytesRef(BinaryRangeUtil.encodeDouble(1D)); + BytesRef encoded6 = new BytesRef(BinaryRangeUtil.encodeDouble(Double.POSITIVE_INFINITY)); assertTrue(encoded1.compareTo(encoded2) < 0); assertTrue(encoded2.compareTo(encoded1) > 0); @@ -53,15 +54,35 @@ public void testBasics() { assertTrue(encoded4.compareTo(encoded3) > 0); assertTrue(encoded4.compareTo(encoded5) < 0); assertTrue(encoded5.compareTo(encoded4) > 0); + assertTrue(encoded5.compareTo(encoded6) < 0); + assertTrue(encoded6.compareTo(encoded5) > 0); + + encoded1 = new BytesRef(BinaryRangeUtil.encodeFloat(Float.NEGATIVE_INFINITY)); + encoded2 = new BytesRef(BinaryRangeUtil.encodeFloat(-1F)); + encoded3 = new BytesRef(BinaryRangeUtil.encodeFloat(-0F)); + encoded4 = new BytesRef(BinaryRangeUtil.encodeFloat(0F)); + encoded5 = new BytesRef(BinaryRangeUtil.encodeFloat(1F)); + encoded6 = new BytesRef(BinaryRangeUtil.encodeFloat(Float.POSITIVE_INFINITY)); + + assertTrue(encoded1.compareTo(encoded2) < 0); + assertTrue(encoded2.compareTo(encoded1) > 0); + assertTrue(encoded2.compareTo(encoded3) < 0); + assertTrue(encoded3.compareTo(encoded2) > 0); + assertTrue(encoded3.compareTo(encoded4) < 0); + assertTrue(encoded4.compareTo(encoded3) > 0); + assertTrue(encoded4.compareTo(encoded5) < 0); + assertTrue(encoded5.compareTo(encoded4) > 0); + assertTrue(encoded5.compareTo(encoded6) < 0); + assertTrue(encoded6.compareTo(encoded5) > 0); } public void testEncode_long() { int iters = randomIntBetween(32, 1024); for (int i = 0; i < iters; i++) { long number1 = randomLong(); - BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encode(number1)); - long number2 = randomLong(); - BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encode(number2)); + BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encodeLong(number1)); + long number2 = randomBoolean() ? number1 + 1 : randomLong(); + BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encodeLong(number2)); int cmp = normalize(Long.compare(number1, number2)); assertEquals(cmp, normalize(encodedNumber1.compareTo(encodedNumber2))); @@ -70,14 +91,48 @@ public void testEncode_long() { } } + public void testVariableLengthEncoding() { + for (int i = -8; i <= 7; ++i) { + assertEquals(1, BinaryRangeUtil.encodeLong(i).length); + } + for (int i = -2048; i <= 2047; ++i) { + if (i < -8 ||i > 7) { + assertEquals(2, BinaryRangeUtil.encodeLong(i).length); + } + } + assertEquals(3, BinaryRangeUtil.encodeLong(-2049).length); + assertEquals(3, BinaryRangeUtil.encodeLong(2048).length); + assertEquals(9, BinaryRangeUtil.encodeLong(Long.MIN_VALUE).length); + assertEquals(9, BinaryRangeUtil.encodeLong(Long.MAX_VALUE).length); + } + public void testEncode_double() { int iters = randomIntBetween(32, 1024); for (int i = 0; i < iters; i++) { double number1 = randomDouble(); - BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encode(number1)); - double number2 = randomDouble(); - BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encode(number2)); + BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encodeDouble(number1)); + double number2 = randomBoolean() ? Math.nextUp(number1) : randomDouble(); + BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encodeDouble(number2)); + + assertEquals(8, encodedNumber1.length); + assertEquals(8, encodedNumber2.length); + int cmp = normalize(Double.compare(number1, number2)); + assertEquals(cmp, normalize(encodedNumber1.compareTo(encodedNumber2))); + cmp = normalize(Double.compare(number2, number1)); + assertEquals(cmp, normalize(encodedNumber2.compareTo(encodedNumber1))); + } + } + + public void testEncode_Float() { + int iters = randomIntBetween(32, 1024); + for (int i = 0; i < iters; i++) { + float number1 = randomFloat(); + BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encodeFloat(number1)); + float number2 = randomBoolean() ? Math.nextUp(number1) : randomFloat(); + BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encodeFloat(number2)); + assertEquals(4, encodedNumber1.length); + assertEquals(4, encodedNumber2.length); int cmp = normalize(Double.compare(number1, number2)); assertEquals(cmp, normalize(encodedNumber1.compareTo(encodedNumber2))); cmp = normalize(Double.compare(number2, number1));