From dffd748747fabf2be412e878db904db907caa6e1 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 14 Jul 2020 14:52:48 -0400 Subject: [PATCH 01/13] Introduce 64-bit unsigned long field type This field type supports - indexing of integer values from [0, 18446744073709551615] - precise queries (term, range) - sorting and aggregations is based on conversion of long values to double and can be imprecise for large values. Closes #32434 --- docs/reference/mapping/types.asciidoc | 4 +- docs/reference/mapping/types/numeric.asciidoc | 3 +- .../mapping/types/unsigned_long.asciidoc | 147 +++++ .../common/xcontent/XContentParser.java | 2 + .../common/xcontent/XContentSubParser.java | 5 + .../xcontent/json/JsonXContentParser.java | 21 + .../support/AbstractXContentParser.java | 20 + .../xcontent/support/MapXContentParser.java | 20 + .../fielddata/IndexNumericFieldData.java | 2 +- .../xcontent/WatcherXContentParser.java | 5 + .../plugin/mapper-unsigned-long/build.gradle | 24 + .../unsignedlong/UnsignedLongFieldMapper.java | 519 ++++++++++++++++++ .../UnsignedLongIndexFieldData.java | 56 ++ .../UnsignedLongLeafFieldData.java | 100 ++++ .../UnsignedLongMapperPlugin.java | 27 + .../UnsignedLongFieldMapperTests.java | 446 +++++++++++++++ .../UnsignedLongFieldTypeTests.java | 151 +++++ .../test/unsigned_long/10_basic.yml | 207 +++++++ .../test/unsigned_long/20_null_value.yml | 79 +++ .../test/unsigned_long/30_multi_fields.yml | 72 +++ .../unsigned_long/40_different_numeric.yml | 102 ++++ 21 files changed, 2009 insertions(+), 3 deletions(-) create mode 100644 docs/reference/mapping/types/unsigned_long.asciidoc create mode 100644 x-pack/plugin/mapper-unsigned-long/build.gradle create mode 100644 x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java create mode 100644 x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java create mode 100644 x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java create mode 100644 x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java create mode 100644 x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java create mode 100644 x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/30_multi_fields.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 5c1671cb30dd3..1af10507793e6 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -9,7 +9,7 @@ document: === Core data types string:: <>, <> and <> -<>:: `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float` +<>:: `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`, <> <>:: `date` <>:: `date_nanos` <>:: `boolean` @@ -136,3 +136,5 @@ include::types/shape.asciidoc[] include::types/constant-keyword.asciidoc[] include::types/wildcard.asciidoc[] + +include::types/unsigned_long.asciidoc[] diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index afd0010135e03..1cc00932d2c02 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -15,6 +15,7 @@ The following numeric types are supported: `float`:: A single-precision 32-bit IEEE 754 floating point number, restricted to finite values. `half_float`:: A half-precision 16-bit IEEE 754 floating point number, restricted to finite values. `scaled_float`:: A floating point number that is backed by a `long`, scaled by a fixed `double` scaling factor. +`unsigned_long`:: An <> with a minimum value of 0 and a maximum value of +2^64^-1+. Below is an example of configuring a mapping with numeric fields: @@ -115,7 +116,7 @@ The following parameters are accepted by numeric types: <>:: Try to convert strings to numbers and truncate fractions for integers. - Accepts `true` (default) and `false`. + Accepts `true` (default) and `false`. Not applicable for unsigned_long. <>:: diff --git a/docs/reference/mapping/types/unsigned_long.asciidoc b/docs/reference/mapping/types/unsigned_long.asciidoc new file mode 100644 index 0000000000000..3a2b0ec3b2420 --- /dev/null +++ b/docs/reference/mapping/types/unsigned_long.asciidoc @@ -0,0 +1,147 @@ +[role="xpack"] +[testenv="basic"] + +[[unsigned-long]] +=== Unsigned long data type +++++ +Unsigned long +++++ + +Unsigned long is a numeric field type that represents an unsigned 64-bit +integer with a minimum value of 0 and a maximum value of +2^64^-1+ +(from 0 to 18446744073709551615). + +At index-time, an indexed value is converted to the singed long range: +[- 9223372036854775808, 9223372036854775807] by subtracting +2^63^+ from it +and stored as a singed long taking 8 bytes. +At query-time, the same conversion is done on query terms. + +[source,console] +-------------------------------------------------- +PUT my_index +{ + "mappings": { + "properties": { + "my_counter": { + "type": "unsigned_long" + } + } + } +} +-------------------------------------------------- + +Unsigned long can be indexed in a numeric or string form, +representing integer values in the range [0, 18446744073709551615]. +They can't have a decimal part. + +[source,console] +-------------------------------- +POST /my_index/_bulk?refresh +{"index":{"_id":1}} +{"my_counter": 0} +{"index":{"_id":2}} +{"my_counter": 9223372036854775808} +{"index":{"_id":3}} +{"my_counter": 18446744073709551614} +{"index":{"_id":4}} +{"my_counter": 18446744073709551615} +-------------------------------- +//TEST[continued] + +Term queries accept any numbers in a numeric or string form. + +[source,console] +-------------------------------- +GET /my_index/_search +{ + "query": { + "term" : { + "my_counter" : 18446744073709551615 + } + } +} +-------------------------------- +//TEST[continued] + +Range queries can contain ranges with decimal parts. +It is recommended to pass ranges as strings to ensure they are parsed +without any loss of precision. + +[source,console] +-------------------------------- +GET /my_index/_search +{ + "query": { + "range" : { + "my_counter" : { + "gte" : "9223372036854775808.5", + "lte" : "18446744073709551615" + } + } + } +} +-------------------------------- +//TEST[continued] + +WARNING: Unlike term and range queries, sorting and aggregations on +unsigned_long data may return imprecise results. For sorting and aggregations +double representation of unsigned longs is used, which means that long values +are first converted to double values. During this conversion, +for long values greater than +2^53^+ there could be some loss of +precision for the least significant digits. Long values less than +2^53^+ +are converted accurately. + +[source,console] +-------------------------------- +GET /my_index/_search +{ + "query": { + "match_all" : {} + }, + "sort" : {"my_counter" : "desc"} <1> +} +-------------------------------- +//TEST[continued] +<1> As both document values: "18446744073709551614" and "18446744073709551615" +are converted to the same double value: "1.8446744073709552E19", this +descending sort may return imprecise results, as the document with a lower +value of "18446744073709551614" may come before the document +with a higher value of "18446744073709551615". + +[[unsigned-long-params]] +==== Parameters for unsigned long fields + +The following parameters are accepted: + +[horizontal] + +<>:: + + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. + +<>:: + + If `true`, malformed numbers are ignored. If `false` (default), malformed + numbers throw an exception and reject the whole document. + +<>:: + + Should the field be searchable? Accepts `true` (default) and `false`. + +<>:: + + Accepts a numeric value of the same `type` as the field which is + substituted for any explicit `null` values. Defaults to `null`, which + means the field is treated as missing. + +<>:: + + Whether the field value should be stored and retrievable separately from + the <> field. Accepts `true` or `false` + (default). + +<>:: + + Metadata about the field. diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index 82a663bd9dc5d..cc4162a0a0439 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -206,6 +206,8 @@ Map map( long longValue() throws IOException; + long unsignedLongValue() throws IOException; + float floatValue() throws IOException; double doubleValue() throws IOException; diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java index 9a8686001e2dc..ad88c10325085 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java @@ -224,6 +224,11 @@ public long longValue() throws IOException { return parser.longValue(); } + @Override + public long unsignedLongValue() throws IOException { + return parser.unsignedLongValue(); + } + @Override public float floatValue() throws IOException { return parser.floatValue(); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java index 7489222df2e76..ca17d923068f4 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java @@ -30,6 +30,7 @@ import org.elasticsearch.core.internal.io.IOUtils; import java.io.IOException; +import java.math.BigInteger; import java.nio.CharBuffer; public class JsonXContentParser extends AbstractXContentParser { @@ -166,6 +167,26 @@ public long doLongValue() throws IOException { return parser.getLongValue(); } + @Override + protected long doUnsignedLongValue() throws IOException { + JsonParser.NumberType numberType = parser.getNumberType(); + if ((numberType == JsonParser.NumberType.INT) || (numberType == JsonParser.NumberType.LONG)) { + long longValue = parser.getLongValue(); + if (longValue < 0) { + throw new IllegalArgumentException("Value [" + longValue + "] is out of range for unsigned long."); + } + return longValue; + } else if (numberType == JsonParser.NumberType.BIG_INTEGER) { + BigInteger bigIntegerValue = parser.getBigIntegerValue(); + if (bigIntegerValue.compareTo(BIGINTEGER_MAX_UNSIGNED_LONG_VALUE) > 0 || bigIntegerValue.compareTo(BigInteger.ZERO) < 0) { + throw new IllegalArgumentException("Value [" + bigIntegerValue + "] is out of range for unsigned long"); + } + return bigIntegerValue.longValue(); + } else { // for all other value types including numbers with decimal parts + throw new IllegalArgumentException("For input string: [" + parser.getValueAsString() + "]."); + } + } + @Override public float doFloatValue() throws IOException { return parser.getFloatValue(); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java index 264af205e488b..f5244329f2e06 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java @@ -46,6 +46,8 @@ public abstract class AbstractXContentParser implements XContentParser { // references to this policy decision throughout the codebase and find // and change any code that needs to apply an alternative policy. public static final boolean DEFAULT_NUMBER_COERCE_POLICY = true; + public static BigInteger BIGINTEGER_MAX_UNSIGNED_LONG_VALUE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); // 2^64 -1 + private static void checkCoerceString(boolean coerce, Class clazz) { if (!coerce) { @@ -208,8 +210,26 @@ public long longValue(boolean coerce) throws IOException { return result; } + @Override + public long unsignedLongValue() throws IOException { + Token token = currentToken(); + if (token == Token.VALUE_STRING) { + return Long.parseUnsignedLong(text()); + } + long result = doUnsignedLongValue(); + return result; + } + protected abstract long doLongValue() throws IOException; + /** + * Returns an unsigned long value of the current numeric token. + * The method must check for proper boundaries: [0; 2^64-1], and also check that it doesn't have a decimal part. + * An exception is raised if any of the conditions is violated. + * Numeric tokens greater than Long.MAX_VALUE must be returned as negative values. + */ + protected abstract long doUnsignedLongValue() throws IOException; + @Override public float floatValue() throws IOException { return floatValue(DEFAULT_NUMBER_COERCE_POLICY); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java index c54e71634d6ed..b2682615c06a5 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java @@ -73,6 +73,26 @@ protected long doLongValue() throws IOException { return numberValue().longValue(); } + @Override + protected long doUnsignedLongValue() throws IOException { + Number value = numberValue(); + if ((value instanceof Integer) || (value instanceof Long) || (value instanceof Short) || (value instanceof Byte)) { + long longValue = value.longValue(); + if (longValue < 0) { + throw new IllegalArgumentException("Value [" + longValue + "] is out of range for unsigned long."); + } + return longValue; + } else if (value instanceof BigInteger) { + BigInteger bigIntegerValue = (BigInteger) value; + if (bigIntegerValue.compareTo(BIGINTEGER_MAX_UNSIGNED_LONG_VALUE) > 0 || bigIntegerValue.compareTo(BigInteger.ZERO) < 0) { + throw new IllegalArgumentException("Value [" + bigIntegerValue + "] is out of range for unsigned long."); + } + return bigIntegerValue.longValue(); + } else { + throw new IllegalArgumentException("For input string: [" + value.toString() + "]."); + } + } + @Override protected float doFloatValue() throws IOException { return numberValue().floatValue(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 6721716bb160e..b319ebf9fd5e7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -87,7 +87,7 @@ public final ValuesSourceType getValuesSourceType() { * Values are casted to the provided targetNumericType type if it doesn't * match the field's numericType. */ - public final SortField sortField( + public SortField sortField( NumericType targetNumericType, Object missingValue, MultiValueMode sortMode, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java index 20b0086c1e4e2..d60d9dc0009cf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java @@ -236,6 +236,11 @@ public long longValue() throws IOException { return parser.longValue(); } + @Override + public long unsignedLongValue() throws IOException { + return parser.unsignedLongValue(); + } + @Override public float floatValue() throws IOException { return parser.floatValue(); diff --git a/x-pack/plugin/mapper-unsigned-long/build.gradle b/x-pack/plugin/mapper-unsigned-long/build.gradle new file mode 100644 index 0000000000000..11e535ab39cbc --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'unsigned-long' + description 'Module for the unsigned long field type' + classname 'org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-unsigned-long' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +integTest.enabled = false diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java new file mode 100644 index 0000000000000..d8dfc096f74a7 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -0,0 +1,519 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.exc.InputCoercionException; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.SimpleMappedFieldType; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class UnsignedLongFieldMapper extends FieldMapper { + protected static long MASK_2_63 = 0x8000000000000000L; + private static BigInteger BIGINTEGER_2_64_MINUS_ONE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); // 2^64 -1 + private static BigDecimal BIGDECIMAL_2_64_MINUS_ONE = new BigDecimal(BIGINTEGER_2_64_MINUS_ONE); + + public static final String CONTENT_TYPE = "unsigned_long"; + // use the same default as numbers + private static final FieldType FIELD_TYPE = new FieldType(); + static { + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + } + + public static class Builder extends FieldMapper.Builder { + + private Boolean ignoreMalformed; + private String nullValue; + + public Builder(String name) { + super(name, FIELD_TYPE); + builder = this; + } + + public Builder ignoreMalformed(boolean ignoreMalformed) { + this.ignoreMalformed = ignoreMalformed; + return builder; + } + + @Override + public Builder indexOptions(IndexOptions indexOptions) { + throw new MapperParsingException("index_options not allowed in field [" + name + "] of type [" + CONTENT_TYPE + "]"); + } + + protected Explicit ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return new Explicit<>(ignoreMalformed, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); + } + return NumberFieldMapper.Defaults.IGNORE_MALFORMED; + } + + public Builder nullValue(String nullValue) { + this.nullValue = nullValue; + return this; + } + + @Override + public UnsignedLongFieldMapper build(BuilderContext context) { + UnsignedLongFieldType type = new UnsignedLongFieldType(buildFullName(context), indexed, hasDocValues, meta); + return new UnsignedLongFieldMapper( + name, + fieldType, + type, + ignoreMalformed(context), + multiFieldsBuilder.build(this, context), + copyTo, + nullValue + ); + } + } + + public static class TypeParser implements Mapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder = new Builder(name); + TypeParsers.parseField(builder, name, node, parserContext); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = entry.getKey(); + Object propNode = entry.getValue(); + if (propName.equals("null_value")) { + if (propNode == null) { + throw new MapperParsingException("Property [null_value] cannot be null."); + } + parseUnsignedLong(propNode); // confirm that null_value is a proper unsigned_long + String nullValue = (propNode instanceof BytesRef) ? ((BytesRef) propNode).utf8ToString() : propNode.toString(); + builder.nullValue(nullValue); + iterator.remove(); + } else if (propName.equals("ignore_malformed")) { + builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode, name + ".ignore_malformed")); + iterator.remove(); + } + } + return builder; + } + } + + public static final class UnsignedLongFieldType extends SimpleMappedFieldType { + + public UnsignedLongFieldType(String name, boolean indexed, boolean hasDocValues, Map meta) { + super(name, indexed, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + } + + public UnsignedLongFieldType(String name) { + this(name, true, true, Collections.emptyMap()); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + failIfNotIndexed(); + Long longValue = parseTerm(value); + if (longValue == null) { + return new MatchNoDocsQuery(); + } + Query query = LongPoint.newExactQuery(name(), convertToSignedLong(longValue)); + if (boost() != 1f) { + query = new BoostQuery(query, boost()); + } + return query; + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + failIfNotIndexed(); + long[] lvalues = new long[values.size()]; + int upTo = 0; + for (int i = 0; i < values.size(); i++) { + Object value = values.get(i); + Long longValue = parseTerm(value); + if (longValue != null) { + lvalues[upTo++] = convertToSignedLong(longValue); + } + } + if (upTo == 0) { + return new MatchNoDocsQuery(); + } + if (upTo != lvalues.length) { + lvalues = Arrays.copyOf(lvalues, upTo); + } + Query query = LongPoint.newSetQuery(name(), lvalues); + if (boost() != 1f) { + query = new BoostQuery(query, boost()); + } + return query; + } + + @Override + public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { + failIfNotIndexed(); + long l = Long.MIN_VALUE; + long u = Long.MAX_VALUE; + if (lowerTerm != null) { + Long lt = parseLowerRangeTerm(lowerTerm, includeLower); + if (lt == null) return new MatchNoDocsQuery(); + l = convertToSignedLong(lt); + } + if (upperTerm != null) { + Long ut = parseUpperRangeTerm(upperTerm, includeUpper); + if (ut == null) return new MatchNoDocsQuery(); + u = convertToSignedLong(ut); + } + if (l > u) return new MatchNoDocsQuery(); + + Query query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + } + if (boost() != 1f) { + query = new BoostQuery(query, boost()); + } + return query; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + failIfNoDocValues(); + return (cache, breakerService, mapperService) -> { + final IndexNumericFieldData signedLongValues = new SortedNumericIndexFieldData.Builder( + name(), + IndexNumericFieldData.NumericType.LONG + ).build(cache, breakerService, mapperService); + return new UnsignedLongIndexFieldData(signedLongValues); + }; + } + + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + return convertToOriginal(((Number) value).longValue()); + } + + @Override + public DocValueFormat docValueFormat(String format, ZoneId timeZone) { + if (timeZone != null) { + throw new IllegalArgumentException( + "Field [" + name() + "] of type [" + typeName() + "] does not support custom time zones" + ); + } + if (format == null) { + return DocValueFormat.RAW; + } else { + return new DocValueFormat.Decimal(format); + } + } + + @Override + public Function pointReaderIfPossible() { + if (isSearchable()) { + return (value) -> LongPoint.decodeDimension(value, 0); + } + return null; + } + + /** + * Parses value to unsigned long for Term Query + * @param value to to parse + * @return parsed value, if a value represents an unsigned long in the range [0, 18446744073709551615] + * null, if a value represents some other number + * throws an exception if a value is wrongly formatted number + */ + protected static Long parseTerm(Object value) { + if (value instanceof Number) { + if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { + long lv = ((Number) value).longValue(); + if (lv >= 0) { + return lv; + } + } else if (value instanceof BigInteger) { + BigInteger bigIntegerValue = (BigInteger) value; + if (bigIntegerValue.compareTo(BigInteger.ZERO) >= 0 && bigIntegerValue.compareTo(BIGINTEGER_2_64_MINUS_ONE) <= 0) { + return bigIntegerValue.longValue(); + } + } + } else { + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + try { + return Long.parseUnsignedLong(stringValue); + } catch (NumberFormatException e) { + // try again in case a number was negative or contained decimal + Double.parseDouble(stringValue); // throws an exception if it is an improper number + } + } + return null; // any other number: decimal or beyond the range of unsigned long + } + + /** + * Parses a lower term for a range query + * @param value to parse + * @param include whether a value should be included + * @return parsed value to long considering include parameter + * 0, if value is less than 0 + * a value truncated to long, if value is in range [0, 18446744073709551615] + * null, if value is higher than the maximum allowed value for unsigned long + * throws an exception is value represents wrongly formatted number + */ + protected static Long parseLowerRangeTerm(Object value, boolean include) { + if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { + long longValue = ((Number) value).longValue(); + if (longValue < 0) return 0L; // limit lowerTerm to min value for unsigned long: 0 + if (include == false) { // start from the next value + // for unsigned long, the next value for Long.MAX_VALUE is -9223372036854775808L + longValue = longValue == Long.MAX_VALUE ? Long.MIN_VALUE : ++longValue; + } + return longValue; + } + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + final BigDecimal bigDecimalValue = new BigDecimal(stringValue); // throws an exception if it is an improper number + if (bigDecimalValue.compareTo(BigDecimal.ZERO) <= 0) { + return 0L; // for values <=0, set lowerTerm to 0 + } + int c = bigDecimalValue.compareTo(BIGDECIMAL_2_64_MINUS_ONE); + if (c > 0 || (c == 0 && include == false)) { + return null; // lowerTerm is beyond maximum value + } + long longValue = bigDecimalValue.longValue(); + boolean hasDecimal = (bigDecimalValue.scale() > 0 && bigDecimalValue.stripTrailingZeros().scale() > 0); + if (include == false || hasDecimal) { + ++longValue; + } + return longValue; + } + + /** + * Parses an upper term for a range query + * @param value to parse + * @param include whether a value should be included + * @return parsed value to long considering include parameter + * null, if value is less that 0, as value is lower than the minimum allowed value for unsigned long + * a value truncated to long if value is in range [0, 18446744073709551615] + * -1 (unsigned long of 18446744073709551615) for values greater than 18446744073709551615 + * throws an exception is value represents wrongly formatted number + */ + protected static Long parseUpperRangeTerm(Object value, boolean include) { + if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { + long longValue = ((Number) value).longValue(); + if ((longValue < 0) || (longValue == 0 && include == false)) return null; // upperTerm is below minimum + longValue = include ? longValue : --longValue; + return longValue; + } + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + final BigDecimal bigDecimalValue = new BigDecimal(stringValue); // throws an exception if it is an improper number + int c = bigDecimalValue.compareTo(BigDecimal.ZERO); + if (c < 0 || (c == 0 && include == false)) { + return null; // upperTerm is below minimum + } + if (bigDecimalValue.compareTo(BIGDECIMAL_2_64_MINUS_ONE) > 0) { + return -1L; // limit upperTerm to max value for unsigned long: 18446744073709551615 + } + long longValue = bigDecimalValue.longValue(); + boolean hasDecimal = (bigDecimalValue.scale() > 0 && bigDecimalValue.stripTrailingZeros().scale() > 0); + if (include == false && hasDecimal == false) { + --longValue; + } + return longValue; + } + } + + private Explicit ignoreMalformed; + private final String nullValue; + private final Long nullValueNumeric; + + private UnsignedLongFieldMapper( + String simpleName, + FieldType fieldType, + UnsignedLongFieldType mappedFieldType, + Explicit ignoreMalformed, + MultiFields multiFields, + CopyTo copyTo, + String nullValue + ) { + super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); + this.nullValue = nullValue; + this.nullValueNumeric = nullValue == null ? null : convertToSignedLong(parseUnsignedLong(nullValue)); + this.ignoreMalformed = ignoreMalformed; + } + + @Override + public UnsignedLongFieldType fieldType() { + return (UnsignedLongFieldType) super.fieldType(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected UnsignedLongFieldMapper clone() { + return (UnsignedLongFieldMapper) super.clone(); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + XContentParser parser = context.parser(); + Long numericValue; + if (context.externalValueSet()) { + numericValue = parseUnsignedLong(context.externalValue()); + } else if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + numericValue = null; + } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0) { + numericValue = null; + } else { + try { + numericValue = parser.unsignedLongValue(); + } catch (InputCoercionException | IllegalArgumentException | JsonParseException e) { + if (ignoreMalformed.value() && parser.currentToken().isValue()) { + context.addIgnoredField(mappedFieldType.name()); + return; + } else { + throw e; + } + } + } + if (numericValue == null) { + numericValue = nullValueNumeric; + if (numericValue == null) return; + } else { + numericValue = convertToSignedLong(numericValue); + } + + boolean docValued = fieldType().hasDocValues(); + boolean indexed = fieldType().isSearchable(); + boolean stored = fieldType.stored(); + + List fields = NumberFieldMapper.NumberType.LONG.createFields(fieldType().name(), numericValue, indexed, docValued, stored); + context.doc().addAll(fields); + if (docValued == false && (indexed || stored)) { + createFieldNamesField(context); + } + } + + @Override + protected void mergeOptions(FieldMapper other, List conflicts) { + UnsignedLongFieldMapper mergeWith = (UnsignedLongFieldMapper) other; + if (mergeWith.ignoreMalformed.explicit()) { + this.ignoreMalformed = mergeWith.ignoreMalformed; + } + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + + if (includeDefaults || ignoreMalformed.explicit()) { + builder.field("ignore_malformed", ignoreMalformed.value()); + } + if (nullValue != null) { + builder.field("null_value", nullValue); + } + } + + /** + * Parse object to unsigned long + * @param value must represent an unsigned long in rage [0;18446744073709551615] or an exception will be thrown + */ + private static long parseUnsignedLong(Object value) { + if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { + long lv = ((Number) value).longValue(); + if (lv < 0) { + throw new IllegalArgumentException("Value [" + lv + "] is out of range for unsigned long."); + } + return lv; + } + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + try { + return Long.parseUnsignedLong(stringValue); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("For input string: \"" + stringValue + "\""); + } + } + + /** + * Convert an unsigned long to the singed long by subtract 2^63 from it + * @param value – unsigned long value in the range [0; 2^64-1], values greater than 2^63-1 are negative + * @return signed long value in the range [-2^63; 2^63-1] + */ + private static long convertToSignedLong(long value) { + // subtracting 2^63 or 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 + // equivalent to flipping the first bit + return value ^ MASK_2_63; + } + + /** + * Convert a signed long to unsigned by adding 2^63 to it + * @param value – signed long value in the range [-2^63; 2^63-1] + * @return unsigned long value in the range [0; 2^64-1], values greater then 2^63-1 are negative + */ + protected static long convertToOriginal(long value) { + // adding 2^63 or 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 + // equivalent to flipping the first bit + return value ^ MASK_2_63; + } + +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java new file mode 100644 index 0000000000000..0802fd4f192ed --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + +public class UnsignedLongIndexFieldData extends IndexNumericFieldData { + private final IndexNumericFieldData signedLongFieldData; + + UnsignedLongIndexFieldData(IndexNumericFieldData signedLongFieldData) { + this.signedLongFieldData = signedLongFieldData; + } + + @Override + public String getFieldName() { + return signedLongFieldData.getFieldName(); + } + + @Override + public ValuesSourceType getValuesSourceType() { + return signedLongFieldData.getValuesSourceType(); + } + + @Override + public LeafNumericFieldData load(LeafReaderContext context) { + return new UnsignedLongLeafFieldData(signedLongFieldData.load(context)); + } + + @Override + public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception { + return new UnsignedLongLeafFieldData(signedLongFieldData.loadDirect(context)); + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + @Override + public void clear() { + signedLongFieldData.clear(); + } + + @Override + public NumericType getNumericType() { + return NumericType.DOUBLE; + } + +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java new file mode 100644 index 0000000000000..5c3df93a1e430 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.elasticsearch.index.fielddata.NumericDoubleValues; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; + +import java.io.IOException; + +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.convertToOriginal; + +public class UnsignedLongLeafFieldData implements LeafNumericFieldData { + private final LeafNumericFieldData signedLongFD; + + UnsignedLongLeafFieldData(LeafNumericFieldData signedLongFD) { + this.signedLongFD = signedLongFD; + } + + @Override + public SortedNumericDocValues getLongValues() { + return FieldData.castToLong(getDoubleValues()); + } + + @Override + public SortedNumericDoubleValues getDoubleValues() { + final SortedNumericDocValues values = signedLongFD.getLongValues(); + final NumericDocValues singleValues = DocValues.unwrapSingleton(values); + if (singleValues != null) { + return FieldData.singleton(new NumericDoubleValues() { + @Override + public boolean advanceExact(int doc) throws IOException { + return singleValues.advanceExact(doc); + } + + @Override + public double doubleValue() throws IOException { + return convertUnsignedLongToDouble(singleValues.longValue()); + } + }); + } else { + return new SortedNumericDoubleValues() { + + @Override + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); + } + + @Override + public double nextValue() throws IOException { + return convertUnsignedLongToDouble(values.nextValue()); + } + + @Override + public int docValueCount() { + return values.docValueCount(); + } + }; + } + } + + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues.Doubles(getDoubleValues()); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return FieldData.toString(getDoubleValues()); + } + + @Override + public long ramBytesUsed() { + return signedLongFD.ramBytesUsed(); + } + + @Override + public void close() { + signedLongFD.close(); + } + + private static double convertUnsignedLongToDouble(long value) { + if (value < 0L) { + return convertToOriginal(value); // add 2 ^ 63 + } else { + // add 2 ^ 63 as a double to make sure there is no overflow and final result is positive + return 0x1.0p63 + value; + } + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java new file mode 100644 index 0000000000000..a3ea313403b53 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Map; + +import static java.util.Collections.singletonMap; + +public class UnsignedLongMapperPlugin extends Plugin implements MapperPlugin { + + public UnsignedLongMapperPlugin(Settings settings) {} + + @Override + public Map getMappers() { + return singletonMap(UnsignedLongFieldMapper.CONTENT_TYPE, new UnsignedLongFieldMapper.TypeParser()); + } + +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java new file mode 100644 index 0000000000000..9c4b2dd453c89 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -0,0 +1,446 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.FieldMapperTestCase; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.termvectors.TermVectorsService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.junit.Before; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.Collection; +import java.util.Set; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.containsString; + +public class UnsignedLongFieldMapperTests extends FieldMapperTestCase { + + IndexService indexService; + DocumentMapperParser parser; + + @Before + public void setup() { + indexService = createIndex("test"); + parser = indexService.mapperService().documentMapperParser(); + } + + @Override + protected Collection> getPlugins() { + return pluginList(UnsignedLongMapperPlugin.class, LocalStateCompositeXPackPlugin.class); + } + + @Override + protected Set unsupportedProperties() { + return Set.of("analyzer", "similarity"); + } + + @Override + protected UnsignedLongFieldMapper.Builder newBuilder() { + return new UnsignedLongFieldMapper.Builder("my_unsigned_long"); + } + + public void testDefaults() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + // test that indexing values as string + { + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(2, fields.length); + IndexableField pointField = fields[0]; + assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); + assertFalse(pointField.fieldType().stored()); + assertEquals(9223372036854775807L, pointField.numericValue().longValue()); + IndexableField dvField = fields[1]; + assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); + assertEquals(9223372036854775807L, dvField.numericValue().longValue()); + assertFalse(dvField.fieldType().stored()); + } + + // test indexing values as integer numbers + { + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "2", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", 9223372036854775807L).endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(2, fields.length); + IndexableField pointField = fields[0]; + assertEquals(-1L, pointField.numericValue().longValue()); + IndexableField dvField = fields[1]; + assertEquals(-1L, dvField.numericValue().longValue()); + } + + // test that indexing values as number with decimal is not allowed + { + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "3", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", 10.5).endObject()), + XContentType.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat(e.getCause().getMessage(), containsString("For input string: [10.5]")); + } + } + + public void testNotIndexed() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("index", false) + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(1, fields.length); + IndexableField dvField = fields[0]; + assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); + assertEquals(9223372036854775807L, dvField.numericValue().longValue()); + } + + public void testNoDocValues() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("doc_values", false) + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(1, fields.length); + IndexableField pointField = fields[0]; + assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); + assertEquals(9223372036854775807L, pointField.numericValue().longValue()); + } + + public void testStore() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("store", true) + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(3, fields.length); + IndexableField pointField = fields[0]; + assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); + assertEquals(9223372036854775807L, pointField.numericValue().longValue()); + IndexableField dvField = fields[1]; + assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); + assertEquals(9223372036854775807L, dvField.numericValue().longValue()); + IndexableField storedField = fields[2]; + assertTrue(storedField.fieldType().stored()); + assertEquals(9223372036854775807L, storedField.numericValue().longValue()); + } + + public void testCoerceMappingParameterIsIllegal() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("coerce", false) + .endObject() + .endObject() + .endObject() + .endObject() + ); + ThrowingRunnable runnable = () -> parser.parse("_doc", new CompressedXContent(mapping)); + ; + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertEquals(e.getMessage(), "Mapping definition for [my_unsigned_long] has unsupported parameters: [coerce : false]"); + } + + public void testNullValue() throws IOException { + // test that if null value is not defined, field is not indexed + { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("my_unsigned_long").endObject()), + XContentType.JSON + ) + ); + assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("my_unsigned_long")); + } + + // test that if null value is defined, it is used + { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("null_value", "18446744073709551615") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("my_unsigned_long").endObject()), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(2, fields.length); + IndexableField pointField = fields[0]; + assertEquals(9223372036854775807L, pointField.numericValue().longValue()); + IndexableField dvField = fields[1]; + assertEquals(9223372036854775807L, dvField.numericValue().longValue()); + } + } + + public void testIgnoreMalformed() throws Exception { + // test ignore_malformed is false by default + { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + Object malformedValue1 = "a"; + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "_doc", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue1).endObject()), + XContentType.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat(e.getCause().getMessage(), containsString("For input string: \"a\"")); + + Object malformedValue2 = Boolean.FALSE; + runnable = () -> mapper.parse( + new SourceToParse( + "test", + "_doc", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue2).endObject()), + XContentType.JSON + ) + ); + e = expectThrows(MapperParsingException.class, runnable); + assertThat(e.getCause().getMessage(), containsString("Current token")); + assertThat(e.getCause().getMessage(), containsString("not numeric, can not use numeric value accessors")); + } + + // test ignore_malformed when set to true ignored malformed documents + { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("ignore_malformed", true) + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + Object malformedValue1 = "a"; + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue1).endObject()), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(0, fields.length); + assertArrayEquals(new String[] { "my_unsigned_long" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored"))); + + Object malformedValue2 = Boolean.FALSE; + ParsedDocument doc2 = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue2).endObject()), + XContentType.JSON + ) + ); + IndexableField[] fields2 = doc2.rootDoc().getFields("my_unsigned_long"); + assertEquals(0, fields2.length); + assertArrayEquals(new String[] { "my_unsigned_long" }, TermVectorsService.getValues(doc2.rootDoc().getFields("_ignored"))); + } + } + + public void testIndexingOutOfRangeValues() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + + for (Object outOfRangeValue : new Object[] { "-1", -1L, "18446744073709551616", new BigInteger("18446744073709551616") }) { + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "_doc", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", outOfRangeValue).endObject()), + XContentType.JSON + ) + ); + expectThrows(MapperParsingException.class, runnable); + } + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java new file mode 100644 index 0000000000000..e3e881c5389c6 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType; +import java.util.List; + +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseTerm; +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseLowerRangeTerm; +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseUpperRangeTerm; + +public class UnsignedLongFieldTypeTests extends FieldTypeTestCase { + + public void testTermQuery() { + UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long"); + + assertEquals(LongPoint.newExactQuery("my_unsigned_long", -9223372036854775808L), ft.termQuery(0, null)); + assertEquals(LongPoint.newExactQuery("my_unsigned_long", 0L), ft.termQuery("9223372036854775808", null)); + assertEquals(LongPoint.newExactQuery("my_unsigned_long", 9223372036854775807L), ft.termQuery("18446744073709551615", null)); + + assertEquals(new MatchNoDocsQuery(), ft.termQuery(-1L, null)); + assertEquals(new MatchNoDocsQuery(), ft.termQuery(10.5, null)); + assertEquals(new MatchNoDocsQuery(), ft.termQuery("18446744073709551616", null)); + + expectThrows(NumberFormatException.class, () -> ft.termQuery("18incorrectnumber", null)); + } + + public void testTermsQuery() { + UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long"); + + assertEquals( + LongPoint.newSetQuery("my_unsigned_long", -9223372036854775808L, 0L, 9223372036854775807L), + ft.termsQuery(List.of("0", "9223372036854775808", "18446744073709551615"), null) + ); + + assertEquals(new MatchNoDocsQuery(), ft.termsQuery(List.of(-9223372036854775808L, -1L), null)); + assertEquals(new MatchNoDocsQuery(), ft.termsQuery(List.of("-0.5", "3.14", "18446744073709551616"), null)); + + expectThrows(NumberFormatException.class, () -> ft.termsQuery(List.of("18incorrectnumber"), null)); + } + + public void testRangeQuery() { + UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long", true, false, null); + + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", -9223372036854775808L, -9223372036854775808L), + ft.rangeQuery(-1L, 0L, true, true, null) + ); + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", -9223372036854775808L, -9223372036854775808L), + ft.rangeQuery(0.0, 0.5, true, true, null) + ); + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", 0, 0), + ft.rangeQuery("9223372036854775807", "9223372036854775808", false, true, null) + ); + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", -9223372036854775808L, 9223372036854775806L), + ft.rangeQuery(null, "18446744073709551614.5", true, true, null) + ); + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", 9223372036854775807L, 9223372036854775807L), + ft.rangeQuery("18446744073709551615", "18446744073709551616", true, true, null) + ); + + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(-1f, -0.5f, true, true, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(-1L, 0L, true, false, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(9223372036854775807L, 9223372036854775806L, true, true, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("18446744073709551616", "18446744073709551616", true, true, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("18446744073709551615", "18446744073709551616", false, true, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(9223372036854775807L, 9223372036854775806L, true, true, null)); + + expectThrows(NumberFormatException.class, () -> ft.rangeQuery("18incorrectnumber", "18incorrectnumber", true, true, null)); + } + + public void testParseTermForTermQuery() { + // values that represent proper unsigned long number + assertEquals(0L, parseTerm("0").longValue()); + assertEquals(0L, parseTerm(0).longValue()); + assertEquals(9223372036854775807L, parseTerm(9223372036854775807L).longValue()); + assertEquals(-1L, parseTerm("18446744073709551615").longValue()); + + // values that represent numbers but not unsigned long and not in range of [0; 18446744073709551615] + assertEquals(null, parseTerm("-9223372036854775808.05")); + assertEquals(null, parseTerm(-9223372036854775808L)); + assertEquals(null, parseTerm(0.0)); + assertEquals(null, parseTerm(0.5)); + assertEquals(null, parseTerm("18446744073709551616")); + + // wrongly formatted numbers + expectThrows(NumberFormatException.class, () -> parseTerm("18incorrectnumber")); + } + + public void testParseLowerTermForRangeQuery() { + // values that are lower than min for lowerTerm are converted to 0 + assertEquals(0L, parseLowerRangeTerm(-9223372036854775808L, true).longValue()); + assertEquals(0L, parseLowerRangeTerm("-9223372036854775808", true).longValue()); + assertEquals(0L, parseLowerRangeTerm("-1", true).longValue()); + assertEquals(0L, parseLowerRangeTerm("-0.5", true).longValue()); + + assertEquals(0L, parseLowerRangeTerm(0L, true).longValue()); + assertEquals(0L, parseLowerRangeTerm("0", true).longValue()); + assertEquals(0L, parseLowerRangeTerm("0.0", true).longValue()); + assertEquals(1L, parseLowerRangeTerm("0.5", true).longValue()); + assertEquals(9223372036854775807L, parseLowerRangeTerm(9223372036854775806L, false).longValue()); + assertEquals(9223372036854775807L, parseLowerRangeTerm(9223372036854775807L, true).longValue()); + assertEquals(-9223372036854775808L, parseLowerRangeTerm(9223372036854775807L, false).longValue()); + assertEquals(-1L, parseLowerRangeTerm("18446744073709551614", false).longValue()); + assertEquals(-1L, parseLowerRangeTerm("18446744073709551614.1", true).longValue()); + assertEquals(-1L, parseLowerRangeTerm("18446744073709551615", true).longValue()); + + // values that are higher than max for lowerTerm don't return results + assertEquals(null, parseLowerRangeTerm("18446744073709551615", false)); + assertEquals(null, parseLowerRangeTerm("18446744073709551616", true)); + + // wrongly formatted numbers + expectThrows(NumberFormatException.class, () -> parseLowerRangeTerm("18incorrectnumber", true)); + } + + public void testParseUpperTermForRangeQuery() { + // values that are lower than min for upperTerm don't return results + assertEquals(null, parseUpperRangeTerm(-9223372036854775808L, true)); + assertEquals(null, parseUpperRangeTerm("-1", true)); + assertEquals(null, parseUpperRangeTerm("-0.5", true)); + assertEquals(null, parseUpperRangeTerm(0L, false)); + + assertEquals(0L, parseUpperRangeTerm(0L, true).longValue()); + assertEquals(0L, parseUpperRangeTerm("0", true).longValue()); + assertEquals(0L, parseUpperRangeTerm("0.0", true).longValue()); + assertEquals(0L, parseUpperRangeTerm("0.5", true).longValue()); + assertEquals(9223372036854775806L, parseUpperRangeTerm(9223372036854775807L, false).longValue()); + assertEquals(9223372036854775807L, parseUpperRangeTerm(9223372036854775807L, true).longValue()); + assertEquals(-2L, parseUpperRangeTerm("18446744073709551614.5", true).longValue()); + assertEquals(-2L, parseUpperRangeTerm("18446744073709551615", false).longValue()); + assertEquals(-1L, parseUpperRangeTerm("18446744073709551615", true).longValue()); + + // values that are higher than max for upperTerm are converted to "18446744073709551615" or -1 in singed representation + assertEquals(-1L, parseUpperRangeTerm("18446744073709551615.8", true).longValue()); + assertEquals(-1L, parseUpperRangeTerm("18446744073709551616", true).longValue()); + + // wrongly formatted numbers + expectThrows(NumberFormatException.class, () -> parseUpperRangeTerm("18incorrectnumber", true)); + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml new file mode 100644 index 0000000000000..98bd40a0ca1ad --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml @@ -0,0 +1,207 @@ +setup: + + - skip: + version: " - 7.99.99" + reason: "unsigned_long was added in 8.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + ul: + type: unsigned_long + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "ul": 0 } + { "index": {"_id" : "2"} } + { "ul": 9223372036854775807 } + { "index": {"_id" : "3"} } + { "ul": 9223372036854775808 } + { "index": {"_id" : "4"} } + { "ul": 18446744073709551614 } + { "index": {"_id" : "5"} } + { "ul": 18446744073709551615 } + +--- +"Exist query": + + - do: + search: + index: test1 + body: + size: 0 + query: + exists: + field: ul + + - match: { "hits.total.value": 5 } + + +--- +"Term query": + + - do: + search: + index: test1 + body: + query: + term: + ul: 0 + - match: { "hits.total.value": 1 } + - match: {hits.hits.0._id: "1" } + + - do: + search: + index: test1 + body: + query: + term: + ul: 18446744073709551615 + - match: { "hits.total.value": 1 } + - match: {hits.hits.0._id: "5" } + + - do: + search: + index: test1 + body: + query: + term: + ul: 18446744073709551616 + - match: { "hits.total.value": 0 } + +--- +"Terms query": + + - do: + search: + index: test1 + body: + size: 0 + query: + terms: + ul: [0, 9223372036854775808, 18446744073709551615] + + - match: { "hits.total.value": 3 } + +--- +"Range query": + + - do: + search: + index: test1 + body: + size: 0 + query: + range: + ul: + gte: 0 + - match: { "hits.total.value": 5 } + + - do: + search: + index: test1 + body: + size: 0 + query: + range: + ul: + gte: 0.5 + - match: { "hits.total.value": 4 } + + - do: + search: + index: test1 + body: + size: 0 + query: + range: + ul: + lte: 18446744073709551615 + - match: { "hits.total.value": 5 } + + - do: + search: + index: test1 + body: + query: + range: + ul: + lte: "18446744073709551614.5" # this must be string, as number gets converted to double with loss of precision + - match: { "hits.total.value": 4 } + +--- +"Sort": + + - do: + search: + index: test1 + body: + sort: [ { ul: asc } ] + + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.0.sort: [0.0] } + # as sort is based on double representation, there is some loss of precision during converting longs to doubles + # thus both 9223372036854775807 and 9223372036854775808 are converted to 9.223372036854776E18 + # both 18446744073709551614 and 18446744073709551615 are converted to 1.8446744073709552E19 + # hence, we can't assert ids for the following sort results: + - match: {hits.hits.1.sort: [9.223372036854776E18] } # could be docs with _id 2 or 3 + - match: {hits.hits.2.sort: [9.223372036854776E18] } # could be docs with _id 2 or 3 + - match: {hits.hits.3.sort: [1.8446744073709552E19] } # could be docs with _id 4 or 5 + - match: {hits.hits.4.sort: [1.8446744073709552E19] } # could be docs with _id 4 or 5 + +--- +"Aggs": + + - do: + search: + index: test1 + body: + size: 0 + aggs: + ul_terms: + terms: + field: ul + - length: { aggregations.ul_terms.buckets: 3 } + - match: { aggregations.ul_terms.buckets.0.key: 9.223372036854776E18 } + - match: { aggregations.ul_terms.buckets.1.key: 1.8446744073709552E19 } + - match: { aggregations.ul_terms.buckets.2.key: 0.0 } + + - do: + search: + index: test1 + body: + size: 0 + aggs: + ul_histogram: + histogram: + field: ul + interval: 9223372036854775808 + - length: { aggregations.ul_histogram.buckets: 3 } + - match: { aggregations.ul_histogram.buckets.0.key: 0.0 } + - match: { aggregations.ul_histogram.buckets.1.key: 9.223372036854776E18 } + - match: { aggregations.ul_histogram.buckets.2.key: 1.8446744073709552E19 } + + - do: + search: + index: test1 + body: + size: 0 + aggs: + ul_range: + range: + field: ul + ranges: [ + { "to": 9223372036854775807 }, + { "from": 9223372036854775807} + ] + - length: { aggregations.ul_range.buckets: 2 } + - match: { aggregations.ul_range.buckets.0.doc_count: 1 } + - match: { aggregations.ul_range.buckets.1.doc_count: 4 } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml new file mode 100644 index 0000000000000..cd30440d24ab7 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml @@ -0,0 +1,79 @@ +--- +"Null value": + - skip: + version: " - 7.99.99" + reason: "unsigned_long was added in 8.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + ul: + type: unsigned_long + null_value: 17446744073709551615 + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "ul": 0 } + { "index": {"_id" : "2"} } + { "ul": null } + { "index": {"_id" : "3"} } + { "ul": ""} + { "index": {"_id" : "4"} } + { "ul": 18446744073709551615 } + { "index": {"_id" : "5"} } + {} + + # term query + - do: + search: + index: test1 + body: + query: + term: + ul: 17446744073709551615 + - match: { "hits.total.value": 2 } + - match: {hits.hits.0._id: "2" } + - match: {hits.hits.1._id: "3" } + + + # asc sort + - do: + search: + index: test1 + body: + sort: { ul : { order: asc, missing : "_last" } } + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.0.sort: [0.0] } + - match: {hits.hits.1._id: "2" } + - match: {hits.hits.1.sort: [1.7446744073709552E19] } + - match: {hits.hits.2._id: "3" } + - match: {hits.hits.2.sort: [1.7446744073709552E19] } + - match: {hits.hits.3._id: "4" } + - match: {hits.hits.3.sort: [1.8446744073709552E19] } + - match: {hits.hits.4._id: "5" } + + # desc sort + - do: + search: + index: test1 + body: + sort: { ul: { order: desc, missing: "_first" } } + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "5" } + - match: {hits.hits.1._id: "4" } + - match: {hits.hits.1.sort: [1.8446744073709552E19] } + - match: {hits.hits.2._id: "2" } + - match: {hits.hits.2.sort: [1.7446744073709552E19] } + - match: {hits.hits.3._id: "3" } + - match: {hits.hits.3.sort: [1.7446744073709552E19] } + - match: {hits.hits.4._id: "1" } + - match: {hits.hits.4.sort: [0.0] } + diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/30_multi_fields.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/30_multi_fields.yml new file mode 100644 index 0000000000000..c7ee1ac4ac76e --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/30_multi_fields.yml @@ -0,0 +1,72 @@ +--- +"Multi keyword and unsigned_long fields": + - skip: + version: " - 7.99.99" + reason: "unsigned_long was added in 8.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + counter: + type: keyword + fields: + ul: + type: unsigned_long + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "counter": 0 } + { "index": {"_id" : "2"} } + { "counter": 9223372036854775808 } + { "index": {"_id" : "3"} } + { "counter": "9223372036854775808" } + { "index": {"_id" : "4"} } + { "counter": 18446744073709551614 } + { "index": {"_id" : "5"} } + { "counter": 18446744073709551615 } + + # term query + - do: + search: + index: test1 + body: + query: + term: + counter.ul: 9223372036854775808 + - match: { "hits.total.value": 2 } + - match: {hits.hits.0._id: "2" } + - match: {hits.hits.1._id: "3" } + + + # asc sort by keyword + - do: + search: + index: test1 + body: + sort: { counter : { order: asc} } + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.1._id: "4" } + - match: {hits.hits.2._id: "5" } + - match: {hits.hits.3._id: "2" } + - match: {hits.hits.4._id: "3" } + + # asc sort by unsigned long + - do: + search: + index: test1 + body: + sort: { counter.ul: { order: asc} } + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.1._id: "2" } + - match: {hits.hits.2._id: "3" } + - match: {hits.hits.3._id: "4" } + - match: {hits.hits.4._id: "5" } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml new file mode 100644 index 0000000000000..ba37f57ee886c --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml @@ -0,0 +1,102 @@ +--- +"Different numeric types": + - skip: + version: " - 7.99.99" + reason: "unsigned_long was added in 8.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + my_counter: + type: double + + - do: + indices.create: + index: test2 + body: + mappings: + properties: + my_counter: + type: long + + - do: + indices.create: + index: test3 + body: + mappings: + properties: + my_counter: + type: unsigned_long + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "my_counter": 0 } + { "index": {"_id" : "2"} } + { "my_counter": 1000000 } + { "index": {"_id" : "3"} } + { "my_counter": 9223372036854775807 } + - do: + bulk: + index: test2 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "my_counter": 0 } + { "index": {"_id" : "2"} } + { "my_counter": 1000000 } + { "index": {"_id" : "3"} } + { "my_counter": 9223372036854775807 } + + - do: + bulk: + index: test3 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "my_counter": 0 } + { "index": {"_id" : "2"} } + { "my_counter": 1000000 } + { "index": {"_id" : "3"} } + { "my_counter": 18446744073709551615 } + + + - do: + search: + index: test* + body: + size: 0 + query: + range: + my_counter: + gte: 0 + - match: { "hits.total.value": 9 } + + - do: + search: + index: test* + body: + size: 0 + query: + range: + my_counter: + gt: 0 + lt: 9223372036854775807 + - match: { "hits.total.value": 3 } + + - do: + search: + index: test* + body: + size: 0 + query: + range: + my_counter: + gte: 9223372036854775807 + - match: { "hits.total.value": 3 } From 7eb2d4acd6597e07f51e9fe695bd5d195eae14d6 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 14 Aug 2020 18:18:59 -0400 Subject: [PATCH 02/13] Address feedback --- docs/reference/mapping/types/numeric.asciidoc | 2 +- .../mapping/types/unsigned_long.asciidoc | 49 ++-- .../common/xcontent/XContentParser.java | 2 - .../common/xcontent/XContentSubParser.java | 5 - .../xcontent/json/JsonXContentParser.java | 21 -- .../support/AbstractXContentParser.java | 20 -- .../xcontent/support/MapXContentParser.java | 20 -- .../java/org/elasticsearch/painless/Def.java | 13 +- .../elasticsearch/painless/DefCastTests.java | 3 +- .../common/io/stream/StreamInput.java | 8 + .../common/io/stream/StreamOutput.java | 10 +- .../elasticsearch/common/lucene/Lucene.java | 7 + .../fielddata/IndexNumericFieldData.java | 2 +- .../elasticsearch/search/DocValueFormat.java | 66 ++++- .../elasticsearch/search/SearchModule.java | 1 + .../search/SearchSortValues.java | 5 +- .../aggregations/bucket/terms/LongTerms.java | 20 +- .../searchafter/SearchAfterBuilder.java | 9 +- .../searchafter/SearchAfterBuilderTests.java | 23 +- .../org/elasticsearch/test/ESTestCase.java | 11 + .../xcontent/WatcherXContentParser.java | 5 - .../plugin/mapper-unsigned-long/build.gradle | 3 +- .../DocValuesWhitelistExtension.java | 51 ++++ .../unsignedlong/UnsignedLongFieldMapper.java | 47 +-- .../UnsignedLongIndexFieldData.java | 16 +- .../UnsignedLongLeafFieldData.java | 4 +- .../UnsignedLongScriptDocValues.java | 67 +++++ ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/unsignedlong/whitelist.txt | 10 + .../UnsignedLongFieldMapperTests.java | 4 +- .../xpack/unsignedlong/UnsignedLongTests.java | 267 ++++++++++++++++++ .../test/unsigned_long/10_basic.yml | 74 +++-- .../test/unsigned_long/20_null_value.yml | 43 +-- .../unsigned_long/40_different_numeric.yml | 71 +++-- .../test/unsigned_long/50_script_values.yml | 110 ++++++++ 35 files changed, 846 insertions(+), 224 deletions(-) create mode 100644 x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java create mode 100644 x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java create mode 100644 x-pack/plugin/mapper-unsigned-long/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt create mode 100644 x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/50_script_values.yml diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index 1cc00932d2c02..c22cef0b82e4c 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -116,7 +116,7 @@ The following parameters are accepted by numeric types: <>:: Try to convert strings to numbers and truncate fractions for integers. - Accepts `true` (default) and `false`. Not applicable for unsigned_long. + Accepts `true` (default) and `false`. Not applicable for `unsigned_long`. <>:: diff --git a/docs/reference/mapping/types/unsigned_long.asciidoc b/docs/reference/mapping/types/unsigned_long.asciidoc index 3a2b0ec3b2420..6863847ba97af 100644 --- a/docs/reference/mapping/types/unsigned_long.asciidoc +++ b/docs/reference/mapping/types/unsigned_long.asciidoc @@ -9,12 +9,7 @@ Unsigned long is a numeric field type that represents an unsigned 64-bit integer with a minimum value of 0 and a maximum value of +2^64^-1+ -(from 0 to 18446744073709551615). - -At index-time, an indexed value is converted to the singed long range: -[- 9223372036854775808, 9223372036854775807] by subtracting +2^63^+ from it -and stored as a singed long taking 8 bytes. -At query-time, the same conversion is done on query terms. +(from 0 to 18446744073709551615 inclusive). [source,console] -------------------------------------------------- @@ -63,7 +58,11 @@ GET /my_index/_search -------------------------------- //TEST[continued] -Range queries can contain ranges with decimal parts. +Range query terms can contain values with decimal parts. +In this case {es} converts them to integer values: +`gte` and `gt` terms are converted to the nearest integer up inclusive, +and `lt` and `lte` ranges are converted to the nearest integer down inclusive. + It is recommended to pass ranges as strings to ensure they are parsed without any loss of precision. @@ -83,13 +82,13 @@ GET /my_index/_search -------------------------------- //TEST[continued] -WARNING: Unlike term and range queries, sorting and aggregations on -unsigned_long data may return imprecise results. For sorting and aggregations -double representation of unsigned longs is used, which means that long values -are first converted to double values. During this conversion, -for long values greater than +2^53^+ there could be some loss of -precision for the least significant digits. Long values less than +2^53^+ -are converted accurately. + +For queries with sort on an `unsigned_long` field, +for a particular document {es} returns a sort value of the type `Long` +if the value of this document is within the range of long values, +or of the type `BigIntger` if the value exceeds this range. + +WARNING: Not all {es} clients can properly handle big integer values. [source,console] -------------------------------- @@ -98,15 +97,25 @@ GET /my_index/_search "query": { "match_all" : {} }, - "sort" : {"my_counter" : "desc"} <1> + "sort" : {"my_counter" : "desc"} } -------------------------------- //TEST[continued] -<1> As both document values: "18446744073709551614" and "18446744073709551615" -are converted to the same double value: "1.8446744073709552E19", this -descending sort may return imprecise results, as the document with a lower -value of "18446744073709551614" may come before the document -with a higher value of "18446744073709551615". + +Similarly to sort values, script values of an `unsigned_long` field +produce `BigInteger` or `Long` values. The same values: `BigInteger` or +`Long` are returned as keys for `terms` aggregation. + +==== Queries with mixed numeric types + +Search queries across several numeric types one of which `unsigned_long` are supported, +except queries with sort. Thus, a sort query across two indexes where the same field +is `unsigned_long` in one index, and `long` in another, doesn't produce correct results +and must be avoided. If there is a need for a such kind of sorting, script based +sorting can be used instead. +Aggregations across several numeric types one of which `unsigned_long` are supported, +except a terms aggregation. + [[unsigned-long-params]] ==== Parameters for unsigned long fields diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index cc4162a0a0439..82a663bd9dc5d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -206,8 +206,6 @@ Map map( long longValue() throws IOException; - long unsignedLongValue() throws IOException; - float floatValue() throws IOException; double doubleValue() throws IOException; diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java index ad88c10325085..9a8686001e2dc 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java @@ -224,11 +224,6 @@ public long longValue() throws IOException { return parser.longValue(); } - @Override - public long unsignedLongValue() throws IOException { - return parser.unsignedLongValue(); - } - @Override public float floatValue() throws IOException { return parser.floatValue(); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java index ca17d923068f4..7489222df2e76 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java @@ -30,7 +30,6 @@ import org.elasticsearch.core.internal.io.IOUtils; import java.io.IOException; -import java.math.BigInteger; import java.nio.CharBuffer; public class JsonXContentParser extends AbstractXContentParser { @@ -167,26 +166,6 @@ public long doLongValue() throws IOException { return parser.getLongValue(); } - @Override - protected long doUnsignedLongValue() throws IOException { - JsonParser.NumberType numberType = parser.getNumberType(); - if ((numberType == JsonParser.NumberType.INT) || (numberType == JsonParser.NumberType.LONG)) { - long longValue = parser.getLongValue(); - if (longValue < 0) { - throw new IllegalArgumentException("Value [" + longValue + "] is out of range for unsigned long."); - } - return longValue; - } else if (numberType == JsonParser.NumberType.BIG_INTEGER) { - BigInteger bigIntegerValue = parser.getBigIntegerValue(); - if (bigIntegerValue.compareTo(BIGINTEGER_MAX_UNSIGNED_LONG_VALUE) > 0 || bigIntegerValue.compareTo(BigInteger.ZERO) < 0) { - throw new IllegalArgumentException("Value [" + bigIntegerValue + "] is out of range for unsigned long"); - } - return bigIntegerValue.longValue(); - } else { // for all other value types including numbers with decimal parts - throw new IllegalArgumentException("For input string: [" + parser.getValueAsString() + "]."); - } - } - @Override public float doFloatValue() throws IOException { return parser.getFloatValue(); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java index f5244329f2e06..264af205e488b 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java @@ -46,8 +46,6 @@ public abstract class AbstractXContentParser implements XContentParser { // references to this policy decision throughout the codebase and find // and change any code that needs to apply an alternative policy. public static final boolean DEFAULT_NUMBER_COERCE_POLICY = true; - public static BigInteger BIGINTEGER_MAX_UNSIGNED_LONG_VALUE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); // 2^64 -1 - private static void checkCoerceString(boolean coerce, Class clazz) { if (!coerce) { @@ -210,26 +208,8 @@ public long longValue(boolean coerce) throws IOException { return result; } - @Override - public long unsignedLongValue() throws IOException { - Token token = currentToken(); - if (token == Token.VALUE_STRING) { - return Long.parseUnsignedLong(text()); - } - long result = doUnsignedLongValue(); - return result; - } - protected abstract long doLongValue() throws IOException; - /** - * Returns an unsigned long value of the current numeric token. - * The method must check for proper boundaries: [0; 2^64-1], and also check that it doesn't have a decimal part. - * An exception is raised if any of the conditions is violated. - * Numeric tokens greater than Long.MAX_VALUE must be returned as negative values. - */ - protected abstract long doUnsignedLongValue() throws IOException; - @Override public float floatValue() throws IOException { return floatValue(DEFAULT_NUMBER_COERCE_POLICY); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java index b2682615c06a5..c54e71634d6ed 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java @@ -73,26 +73,6 @@ protected long doLongValue() throws IOException { return numberValue().longValue(); } - @Override - protected long doUnsignedLongValue() throws IOException { - Number value = numberValue(); - if ((value instanceof Integer) || (value instanceof Long) || (value instanceof Short) || (value instanceof Byte)) { - long longValue = value.longValue(); - if (longValue < 0) { - throw new IllegalArgumentException("Value [" + longValue + "] is out of range for unsigned long."); - } - return longValue; - } else if (value instanceof BigInteger) { - BigInteger bigIntegerValue = (BigInteger) value; - if (bigIntegerValue.compareTo(BIGINTEGER_MAX_UNSIGNED_LONG_VALUE) > 0 || bigIntegerValue.compareTo(BigInteger.ZERO) < 0) { - throw new IllegalArgumentException("Value [" + bigIntegerValue + "] is out of range for unsigned long."); - } - return bigIntegerValue.longValue(); - } else { - throw new IllegalArgumentException("For input string: [" + value.toString() + "]."); - } - } - @Override protected float doFloatValue() throws IOException { return numberValue().floatValue(); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java index 8f42acc0e7047..dd7e00a6c8a2b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java @@ -29,6 +29,7 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; +import java.math.BigInteger; import java.time.ZonedDateTime; import java.util.BitSet; import java.util.Collections; @@ -734,6 +735,8 @@ public static double defTodoubleImplicit(final Object value) { return (float)value; } else if (value instanceof Double) { return (double)value; + } else if (value instanceof BigInteger) { + return ((BigInteger)value).doubleValue(); } else { throw new ClassCastException("cannot implicitly cast " + "def [" + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName() + "] to " + @@ -866,7 +869,8 @@ public static double defTodoubleExplicit(final Object value) { value instanceof Integer || value instanceof Long || value instanceof Float || - value instanceof Double + value instanceof Double || + value instanceof BigInteger ) { return ((Number)value).doubleValue(); } else { @@ -1004,7 +1008,9 @@ public static Double defToDoubleImplicit(final Object value) { } else if (value instanceof Float) { return (double)(float)value; } else if (value instanceof Double) { - return (Double)value; + return (Double) value; + } else if (value instanceof BigInteger) { + return ((BigInteger)value).doubleValue(); } else { throw new ClassCastException("cannot implicitly cast " + "def [" + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName() + "] to " + @@ -1151,7 +1157,8 @@ public static Double defToDoubleExplicit(final Object value) { value instanceof Integer || value instanceof Long || value instanceof Float || - value instanceof Double + value instanceof Double || + value instanceof BigInteger ) { return ((Number)value).doubleValue(); } else { diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefCastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefCastTests.java index 39ab7b004bea1..78bf947753e44 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefCastTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefCastTests.java @@ -166,6 +166,7 @@ public void testdefTodoubleImplicit() { assertEquals((double)0, exec("def d = Long.valueOf(0); double b = d; b")); assertEquals((double)0, exec("def d = Float.valueOf(0); double b = d; b")); assertEquals((double)0, exec("def d = Double.valueOf(0); double b = d; b")); + assertEquals((double)0, exec("def d = BigInteger.valueOf(0); double b = d; b")); expectScriptThrows(ClassCastException.class, () -> exec("def d = new ArrayList(); double b = d;")); } @@ -485,7 +486,7 @@ public void testdefToFloatImplicit() { expectScriptThrows(ClassCastException.class, () -> exec("def d = Double.valueOf(0); Float b = d;")); expectScriptThrows(ClassCastException.class, () -> exec("def d = new ArrayList(); Float b = d;")); } - + public void testdefToDoubleImplicit() { expectScriptThrows(ClassCastException.class, () -> exec("def d = 'string'; Double b = d;")); expectScriptThrows(ClassCastException.class, () -> exec("def d = true; Double b = d;")); diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 21b89d1446fbd..27a677484d20d 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -51,6 +51,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.math.BigInteger; import java.nio.file.AccessDeniedException; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.DirectoryNotEmptyException; @@ -348,6 +349,11 @@ public Long readOptionalLong() throws IOException { return null; } + public BigInteger readBigInteger() throws IOException { + return new BigInteger(readString()); + } + + @Nullable public Text readOptionalText() throws IOException { int length = readInt(); @@ -760,6 +766,8 @@ public Object readGenericValue() throws IOException { return readCollection(StreamInput::readGenericValue, LinkedHashSet::new, Collections.emptySet()); case 25: return readCollection(StreamInput::readGenericValue, HashSet::new, Collections.emptySet()); + case 26: + return readBigInteger(); default: throw new IOException("Can't read unknown type [" + type + "]"); } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index cd25b790da7d5..0c88ee22b7392 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -49,6 +49,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; +import java.math.BigInteger; import java.nio.file.AccessDeniedException; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.DirectoryNotEmptyException; @@ -826,7 +827,14 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep o.writeByte((byte) 25); } o.writeCollection((Set) v, StreamOutput::writeGenericValue); - } + }), + entry( + // TODO: improve serialization of BigInteger + BigInteger.class, + (o, v) -> { + o.writeByte((byte) 26); + o.writeString(v.toString()); + } )); private static Class getGenericType(Object value) { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 18b4d303f0adb..f477c15135d31 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -95,6 +95,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import java.io.IOException; +import java.math.BigInteger; import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; @@ -366,6 +367,8 @@ public static FieldDoc readFieldDoc(StreamInput in) throws IOException { cFields[j] = in.readBoolean(); } else if (type == 9) { cFields[j] = in.readBytesRef(); + } else if (type == 10) { + cFields[j] = new BigInteger(in.readString()); } else { throw new IOException("Can't match type [" + type + "]"); } @@ -510,6 +513,10 @@ public static void writeSortValue(StreamOutput out, Object field) throws IOExcep } else if (type == BytesRef.class) { out.writeByte((byte) 9); out.writeBytesRef((BytesRef) field); + } else if (type == BigInteger.class) { + //TODO: improve serialization of BigInteger + out.writeByte((byte) 10); + out.writeString(field.toString()); } else { throw new IOException("Can't handle sort field value of type [" + type + "]"); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index b319ebf9fd5e7..6721716bb160e 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -87,7 +87,7 @@ public final ValuesSourceType getValuesSourceType() { * Values are casted to the provided targetNumericType type if it doesn't * match the field's numericType. */ - public SortField sortField( + public final SortField sortField( NumericType targetNumericType, Object missingValue, MultiValueMode sortMode, diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 5b45cd660697a..0c5c0483f27aa 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -34,6 +34,7 @@ import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import java.io.IOException; +import java.math.BigInteger; import java.net.InetAddress; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; @@ -48,6 +49,8 @@ /** A formatter for values as returned by the fielddata/doc-values APIs. */ public interface DocValueFormat extends NamedWriteable { + long MASK_2_63 = 0x8000000000000000L; + BigInteger BIGINTEGER_2_64_MINUS_ONE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); // 2^64 -1 /** Format a long value. This is used by terms and histogram aggregations * to format keys for fields that use longs as a doc value representation @@ -467,5 +470,66 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(pattern); } - } + }; + + /** + * DocValues format for unsigned 64 bit long values, + * that are stored as shifted signed 64 bit long values. + */ + DocValueFormat UNSIGNED_LONG_SHIFTED = new DocValueFormat() { + + @Override + public String getWriteableName() { + return "unsigned_long_shifted"; + } + + @Override + public void writeTo(StreamOutput out) { + } + + @Override + public String toString() { + return "unsigned_long_shifted"; + } + + /** + * Formats the unsigned long to the shifted long format + */ + @Override + public long parseLong(String value, boolean roundUp, LongSupplier now) { + long parsedValue = Long.parseUnsignedLong(value); + // subtract 2^63 or 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 + // equivalent to flipping the first bit + return parsedValue ^ MASK_2_63; + } + + /** + * Formats a raw docValue that is stored in the shifted long format to the unsigned long representation. + */ + @Override + public Object format(long value) { + // add 2^63 or 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000, + // equivalent to flipping the first bit + long formattedValue = value ^ MASK_2_63; + if (formattedValue >= 0) { + return formattedValue; + } else { + return BigInteger.valueOf(formattedValue).and(BIGINTEGER_2_64_MINUS_ONE); + } + } + + /** + * Double docValues of the unsigned_long field type are already in the formatted representation, + * so we don't need to do anything here + */ + @Override + public Double format(double value) { + return value; + } + + @Override + public double parseDouble(String value, boolean roundUp, LongSupplier now) { + return Double.parseDouble(value); + } + }; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index ff5a01254fa4c..8b182ea398068 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -698,6 +698,7 @@ private void registerValueFormats() { registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP); registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW); registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY); + registerValueFormat(DocValueFormat.UNSIGNED_LONG_SHIFTED.getWriteableName(), in -> DocValueFormat.UNSIGNED_LONG_SHIFTED); } /** diff --git a/server/src/main/java/org/elasticsearch/search/SearchSortValues.java b/server/src/main/java/org/elasticsearch/search/SearchSortValues.java index d944e9bd89486..85905e67837f8 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchSortValues.java +++ b/server/src/main/java/org/elasticsearch/search/SearchSortValues.java @@ -56,10 +56,13 @@ public SearchSortValues(Object[] rawSortValues, DocValueFormat[] sortValueFormat this.rawSortValues = rawSortValues; this.formattedSortValues = Arrays.copyOf(rawSortValues, rawSortValues.length); for (int i = 0; i < rawSortValues.length; ++i) { - //we currently format only BytesRef but we may want to change that in the future Object sortValue = rawSortValues[i]; if (sortValue instanceof BytesRef) { this.formattedSortValues[i] = sortValueFormats[i].format((BytesRef) sortValue); + } else if ((sortValue instanceof Long) && (sortValueFormats[i] == DocValueFormat.UNSIGNED_LONG_SHIFTED)) { + this.formattedSortValues[i] = sortValueFormats[i].format((Long) sortValue); + } else { + this.formattedSortValues[i] = sortValue; } } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java index 8b3a1d00a678b..de606ad0de62f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java @@ -67,12 +67,20 @@ public String getKeyAsString() { @Override public Object getKey() { - return term; + if (format == DocValueFormat.UNSIGNED_LONG_SHIFTED) { + return format.format(term); + } else { + return term; + } } @Override public Number getKeyAsNumber() { - return term; + if (format == DocValueFormat.UNSIGNED_LONG_SHIFTED) { + return (Number) format.format(term); + } else { + return term; + } } @Override @@ -82,8 +90,12 @@ public int compareKey(Bucket other) { @Override protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { - builder.field(CommonFields.KEY.getPreferredName(), term); - if (format != DocValueFormat.RAW) { + if (format == DocValueFormat.UNSIGNED_LONG_SHIFTED) { + builder.field(CommonFields.KEY.getPreferredName(), format.format(term)); + } else { + builder.field(CommonFields.KEY.getPreferredName(), term); + } + if (format != DocValueFormat.RAW && format != DocValueFormat.UNSIGNED_LONG_SHIFTED) { builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term).toString()); } return builder; diff --git a/server/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java b/server/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java index 6c3ac160bc661..533d87e618113 100644 --- a/server/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java @@ -40,6 +40,7 @@ import org.elasticsearch.search.sort.SortAndFormats; import java.io.IOException; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -91,6 +92,7 @@ public SearchAfterBuilder setSortValues(Object[] values) { if (values[i] instanceof Double) continue; if (values[i] instanceof Float) continue; if (values[i] instanceof Boolean) continue; + if (values[i] instanceof BigInteger) continue; throw new IllegalArgumentException("Can't handle " + SEARCH_AFTER + " field value of type [" + values[i].getClass() + "]"); } sortValues = new Object[values.length]; @@ -181,7 +183,8 @@ private static Object convertValueFromSortType(String fieldName, SortField.Type return Double.parseDouble(value.toString()); case LONG: - if (value instanceof Number) { + // for unsigned_long field type we want to pass search_after value through formatting + if (value instanceof Number && format != DocValueFormat.UNSIGNED_LONG_SHIFTED) { return ((Number) value).longValue(); } return format.parseLong(value.toString(), false, @@ -243,6 +246,10 @@ public static SearchAfterBuilder fromXContent(XContentParser parser) throws IOEx values.add(parser.floatValue()); break; + case BIG_INTEGER: + values.add(parser.text()); + break; + default: throw new IllegalArgumentException("[search_after] does not accept numbers of type [" + parser.numberType() + "], got " + parser.text()); diff --git a/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java index 03a4e2e0a1581..45da49e5f4c48 100644 --- a/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java @@ -43,7 +43,6 @@ import java.io.IOException; import java.math.BigDecimal; -import java.math.BigInteger; import java.util.Collections; import static org.elasticsearch.search.searchafter.SearchAfterBuilder.extractSortType; @@ -59,7 +58,7 @@ private static SearchAfterBuilder randomSearchAfterBuilder() throws IOException SearchAfterBuilder searchAfterBuilder = new SearchAfterBuilder(); Object[] values = new Object[numSearchFrom]; for (int i = 0; i < numSearchFrom; i++) { - int branch = randomInt(9); + int branch = randomInt(10); switch (branch) { case 0: values[i] = randomInt(); @@ -91,6 +90,9 @@ private static SearchAfterBuilder randomSearchAfterBuilder() throws IOException case 9: values[i] = null; break; + case 10: + values[i] = randomBigInteger(); + break; } } searchAfterBuilder.setSortValues(values); @@ -196,27 +198,12 @@ public void testFromXContent() throws Exception { public void testFromXContentIllegalType() throws Exception { for (XContentType type : XContentType.values()) { - // BIG_INTEGER - XContentBuilder xContent = XContentFactory.contentBuilder(type); - xContent.startObject() - .startArray("search_after") - .value(new BigInteger("9223372036854776000")) - .endArray() - .endObject(); - try (XContentParser parser = createParser(xContent)) { - parser.nextToken(); - parser.nextToken(); - parser.nextToken(); - IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> SearchAfterBuilder.fromXContent(parser)); - assertThat(exc.getMessage(), containsString("BIG_INTEGER")); - } - // BIG_DECIMAL // ignore json and yaml, they parse floating point numbers as floats/doubles if (type == XContentType.JSON || type == XContentType.YAML) { continue; } - xContent = XContentFactory.contentBuilder(type); + XContentBuilder xContent = XContentFactory.contentBuilder(type); xContent.startObject() .startArray("search_after") .value(new BigDecimal("9223372036854776003.3")) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 2eaae7792b2a7..09d9d23b500f1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -116,6 +116,7 @@ import java.io.IOException; import java.io.InputStream; +import java.math.BigInteger; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.file.Path; @@ -657,6 +658,16 @@ public static long randomLong() { return random().nextLong(); } + /** + * Returns a random BigInteger uniformly distributed over the range 0 to (2^64 - 1) inclusive + * Currently BigIntegers are only used for unsigned_long field type, where the max value is 2^64 - 1. + * Modify this random generator if a wider range for BigIntegers is necessary. + * @return a random bigInteger in the range [0 ; 2^64 - 1] + */ + public static BigInteger randomBigInteger() { + return new BigInteger(64, random()); + } + /** A random integer from 0..max (inclusive). */ public static int randomInt(int max) { return RandomizedTest.randomInt(max); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java index d60d9dc0009cf..20b0086c1e4e2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java @@ -236,11 +236,6 @@ public long longValue() throws IOException { return parser.longValue(); } - @Override - public long unsignedLongValue() throws IOException { - return parser.unsignedLongValue(); - } - @Override public float floatValue() throws IOException { return parser.floatValue(); diff --git a/x-pack/plugin/mapper-unsigned-long/build.gradle b/x-pack/plugin/mapper-unsigned-long/build.gradle index 11e535ab39cbc..e0eaca675756a 100644 --- a/x-pack/plugin/mapper-unsigned-long/build.gradle +++ b/x-pack/plugin/mapper-unsigned-long/build.gradle @@ -12,11 +12,12 @@ esplugin { name 'unsigned-long' description 'Module for the unsigned long field type' classname 'org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['x-pack-core', 'lang-painless'] } archivesBaseName = 'x-pack-unsigned-long' dependencies { + compileOnly project(':modules:lang-painless:spi') compileOnly project(path: xpackModule('core'), configuration: 'default') testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java new file mode 100644 index 0000000000000..e596b2735fe7b --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.script.BucketAggregationSelectorScript; +import org.elasticsearch.script.FieldScript; +import org.elasticsearch.script.FilterScript; +import org.elasticsearch.script.NumberSortScript; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.StringSortScript; + +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; + +public class DocValuesWhitelistExtension implements PainlessExtension { + + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(DocValuesWhitelistExtension.class, "whitelist.txt"); + + @Override + public Map, List> getContextWhitelists() { + List whitelist = singletonList(WHITELIST); + Map, List> contexts = Map.of( + FieldScript.CONTEXT, + whitelist, + ScoreScript.CONTEXT, + whitelist, + FilterScript.CONTEXT, + whitelist, + AggregationScript.CONTEXT, + whitelist, + NumberSortScript.CONTEXT, + whitelist, + StringSortScript.CONTEXT, + whitelist, + BucketAggregationSelectorScript.CONTEXT, + whitelist + ); + return contexts; + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index d8dfc096f74a7..4d083664a12c6 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -251,7 +251,7 @@ public Object valueForDisplay(Object value) { if (value == null) { return null; } - return convertToOriginal(((Number) value).longValue()); + return value; } @Override @@ -261,11 +261,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { "Field [" + name() + "] of type [" + typeName() + "] does not support custom time zones" ); } - if (format == null) { - return DocValueFormat.RAW; - } else { - return new DocValueFormat.Decimal(format); - } + return DocValueFormat.UNSIGNED_LONG_SHIFTED; } @Override @@ -426,7 +422,11 @@ protected void parseCreateField(ParseContext context) throws IOException { numericValue = null; } else { try { - numericValue = parser.unsignedLongValue(); + if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { + numericValue = parseUnsignedLong(parser.numberValue()); + } else { + numericValue = parseUnsignedLong(parser.text()); + } } catch (InputCoercionException | IllegalArgumentException | JsonParseException e) { if (ignoreMalformed.value() && parser.currentToken().isValue()) { context.addIgnoredField(mappedFieldType.name()); @@ -479,18 +479,29 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, * @param value must represent an unsigned long in rage [0;18446744073709551615] or an exception will be thrown */ private static long parseUnsignedLong(Object value) { - if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { - long lv = ((Number) value).longValue(); - if (lv < 0) { - throw new IllegalArgumentException("Value [" + lv + "] is out of range for unsigned long."); + if (value instanceof Number) { + if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { + long lv = ((Number) value).longValue(); + if (lv < 0) { + throw new IllegalArgumentException("Value [" + lv + "] is out of range for unsigned long."); + } + return lv; + } else if (value instanceof BigInteger) { + BigInteger bigIntegerValue = (BigInteger) value; + if (bigIntegerValue.compareTo(BIGINTEGER_2_64_MINUS_ONE) > 0 || bigIntegerValue.compareTo(BigInteger.ZERO) < 0) { + throw new IllegalArgumentException("Value [" + bigIntegerValue + "] is out of range for unsigned long"); + } + return bigIntegerValue.longValue(); + } + // throw exception for all other numeric types with decimal parts + throw new IllegalArgumentException("For input string: [" + value.toString() + "]."); + } else { + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + try { + return Long.parseUnsignedLong(stringValue); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("For input string: \"" + stringValue + "\""); } - return lv; - } - String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); - try { - return Long.parseUnsignedLong(stringValue); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("For input string: \"" + stringValue + "\""); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java index 0802fd4f192ed..8322aeb2c7da2 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java @@ -12,30 +12,30 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; public class UnsignedLongIndexFieldData extends IndexNumericFieldData { - private final IndexNumericFieldData signedLongFieldData; + private final IndexNumericFieldData signedLongIFD; UnsignedLongIndexFieldData(IndexNumericFieldData signedLongFieldData) { - this.signedLongFieldData = signedLongFieldData; + this.signedLongIFD = signedLongFieldData; } @Override public String getFieldName() { - return signedLongFieldData.getFieldName(); + return signedLongIFD.getFieldName(); } @Override public ValuesSourceType getValuesSourceType() { - return signedLongFieldData.getValuesSourceType(); + return signedLongIFD.getValuesSourceType(); } @Override public LeafNumericFieldData load(LeafReaderContext context) { - return new UnsignedLongLeafFieldData(signedLongFieldData.load(context)); + return new UnsignedLongLeafFieldData(signedLongIFD.load(context)); } @Override public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception { - return new UnsignedLongLeafFieldData(signedLongFieldData.loadDirect(context)); + return new UnsignedLongLeafFieldData(signedLongIFD.loadDirect(context)); } @Override @@ -45,12 +45,12 @@ protected boolean sortRequiresCustomComparator() { @Override public void clear() { - signedLongFieldData.clear(); + signedLongIFD.clear(); } @Override public NumericType getNumericType() { - return NumericType.DOUBLE; + return NumericType.LONG; } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java index 5c3df93a1e430..ef6c5643d88b8 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -29,7 +29,7 @@ public class UnsignedLongLeafFieldData implements LeafNumericFieldData { @Override public SortedNumericDocValues getLongValues() { - return FieldData.castToLong(getDoubleValues()); + return signedLongFD.getLongValues(); } @Override @@ -71,7 +71,7 @@ public int docValueCount() { @Override public ScriptDocValues getScriptValues() { - return new ScriptDocValues.Doubles(getDoubleValues()); + return new UnsignedLongScriptDocValues(getLongValues()); } @Override diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java new file mode 100644 index 0000000000000..7b6e3329769a5 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; + +public class UnsignedLongScriptDocValues extends ScriptDocValues { + private final SortedNumericDocValues in; + private Number[] values = new Number[0]; + private int count; + + /** + * Standard constructor. + */ + public UnsignedLongScriptDocValues(SortedNumericDocValues in) { + this.in = in; + } + + @Override + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = (Number) DocValueFormat.UNSIGNED_LONG_SHIFTED.format(in.nextValue()); + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected void resize(int newSize) { + count = newSize; + values = ArrayUtil.grow(values, count); + } + + public Number getValue() { + return get(0); + } + + @Override + public Number get(int index) { + if (count == 0) { + throw new IllegalStateException( + "A document doesn't have a value for a field! Use doc[].size()==0 to check if a document is missing a field!" + ); + } + return values[index]; + } + + @Override + public int size() { + return count; + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/mapper-unsigned-long/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..fd3625f1872ad --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.unsignedlong.DocValuesWhitelistExtension diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt b/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt new file mode 100644 index 0000000000000..ea0fe0a395c37 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt @@ -0,0 +1,10 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +class org.elasticsearch.xpack.unsignedlong.UnsignedLongScriptDocValues { + Number get(int) + Number getValue() +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 9c4b2dd453c89..1e32b7be7fee5 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -257,7 +257,6 @@ public void testCoerceMappingParameterIsIllegal() throws Exception { .endObject() ); ThrowingRunnable runnable = () -> parser.parse("_doc", new CompressedXContent(mapping)); - ; MapperParsingException e = expectThrows(MapperParsingException.class, runnable); assertEquals(e.getMessage(), "Mapping definition for [my_unsigned_long] has unsupported parameters: [coerce : false]"); } @@ -366,8 +365,7 @@ public void testIgnoreMalformed() throws Exception { ) ); e = expectThrows(MapperParsingException.class, runnable); - assertThat(e.getCause().getMessage(), containsString("Current token")); - assertThat(e.getCause().getMessage(), containsString("not numeric, can not use numeric value accessors")); + assertThat(e.getCause().getMessage(), containsString("For input string: \"false\"")); } // test ignore_malformed when set to true ignored malformed documents diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java new file mode 100644 index 0000000000000..213cf64751329 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java @@ -0,0 +1,267 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.unsignedlong; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.range.Range; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESIntegTestCase; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.range; +import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.containsString; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; + +@ESIntegTestCase.SuiteScopeTestCase + +public class UnsignedLongTests extends ESIntegTestCase { + final int numDocs = 10; + final Number[] values = { + 0L, + 0L, + 100L, + 9223372036854775807L, + new BigInteger("9223372036854775808"), + new BigInteger("10446744073709551613"), + new BigInteger("18446744073709551614"), + new BigInteger("18446744073709551614"), + new BigInteger("18446744073709551615"), + new BigInteger("18446744073709551615") }; + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(UnsignedLongMapperPlugin.class); + } + + @Override + public void setupSuiteScopeCluster() throws Exception { + Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", 3); + prepareCreate("idx").setMapping("ul_field", "type=unsigned_long").setSettings(settings).get(); + List builders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + builders.add(client().prepareIndex("idx").setSource(jsonBuilder().startObject().field("ul_field", values[i]).endObject())); + } + indexRandom(true, builders); + ensureSearchable(); + } + + public void testSort() { + // asc sort + { + SearchResponse response = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .addSort("ul_field", SortOrder.ASC) + .get(); + assertSearchResponse(response); + SearchHit[] hits = response.getHits().getHits(); + assertEquals(hits.length, numDocs); + int i = 0; + for (SearchHit hit : hits) { + assertEquals(values[i++], hit.getSortValues()[0]); + } + } + // desc sort + { + SearchResponse response = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .addSort("ul_field", SortOrder.DESC) + .get(); + assertSearchResponse(response); + SearchHit[] hits = response.getHits().getHits(); + assertEquals(hits.length, numDocs); + int i = numDocs - 1; + for (SearchHit hit : hits) { + assertEquals(values[i--], hit.getSortValues()[0]); + } + } + // asc sort with search_after as Long + { + SearchResponse response = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .addSort("ul_field", SortOrder.ASC) + .searchAfter(new Long[] { 100L }) + .get(); + assertSearchResponse(response); + SearchHit[] hits = response.getHits().getHits(); + assertEquals(hits.length, 7); + int i = 3; + for (SearchHit hit : hits) { + assertEquals(values[i++], hit.getSortValues()[0]); + } + } + // asc sort with search_after as BigInteger + { + SearchResponse response = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .addSort("ul_field", SortOrder.ASC) + .searchAfter(new BigInteger[] { new BigInteger("18446744073709551614") }) + .get(); + assertSearchResponse(response); + SearchHit[] hits = response.getHits().getHits(); + assertEquals(hits.length, 2); + int i = 8; + for (SearchHit hit : hits) { + assertEquals(values[i++], hit.getSortValues()[0]); + } + } + // asc sort with search_after as BigInteger in String format + { + SearchResponse response = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .addSort("ul_field", SortOrder.ASC) + .searchAfter(new String[] { "18446744073709551614" }) + .get(); + assertSearchResponse(response); + SearchHit[] hits = response.getHits().getHits(); + assertEquals(hits.length, 2); + int i = 8; + for (SearchHit hit : hits) { + assertEquals(values[i++], hit.getSortValues()[0]); + } + } + // asc sort with search_after of negative value should fail + { + SearchRequestBuilder srb = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .addSort("ul_field", SortOrder.ASC) + .searchAfter(new Long[] { -1L }); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> srb.get()); + assertThat(exception.getCause().getMessage(), containsString("Failed to parse search_after value")); + } + // asc sort with search_after of value>=2^64 should fail + { + SearchRequestBuilder srb = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .addSort("ul_field", SortOrder.ASC) + .searchAfter(new BigInteger[] { new BigInteger("18446744073709551616") }); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> srb.get()); + assertThat(exception.getCause().getMessage(), containsString("Failed to parse search_after value")); + } + // desc sort with search_after as BigInteger + { + SearchResponse response = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(numDocs) + .addSort("ul_field", SortOrder.DESC) + .searchAfter(new BigInteger[] { new BigInteger("18446744073709551615") }) + .get(); + assertSearchResponse(response); + SearchHit[] hits = response.getHits().getHits(); + assertEquals(hits.length, 8); + int i = 7; + for (SearchHit hit : hits) { + assertEquals(values[i--], hit.getSortValues()[0]); + } + } + } + + public void testAggs() { + // terms agg + { + SearchResponse response = client().prepareSearch("idx").setSize(0).addAggregation(terms("ul_terms").field("ul_field")).get(); + assertSearchResponse(response); + Terms terms = response.getAggregations().get("ul_terms"); + + long[] expectedBucketDocCounts = { 2, 2, 2, 1, 1, 1, 1 }; + Object[] expectedBucketKeys = { + 0L, + new BigInteger("18446744073709551614"), + new BigInteger("18446744073709551615"), + 100L, + 9223372036854775807L, + new BigInteger("9223372036854775808"), + new BigInteger("10446744073709551613") }; + int i = 0; + for (Terms.Bucket bucket : terms.getBuckets()) { + assertEquals(expectedBucketDocCounts[i], bucket.getDocCount()); + assertEquals(expectedBucketKeys[i], bucket.getKey()); + i++; + } + } + + // histogram agg + { + SearchResponse response = client().prepareSearch("idx") + .setSize(0) + .addAggregation(histogram("ul_histo").field("ul_field").interval(9.223372036854776E18).minDocCount(0)) + .get(); + assertSearchResponse(response); + Histogram histo = response.getAggregations().get("ul_histo"); + + long[] expectedBucketDocCounts = { 3, 3, 4 }; + double[] expectedBucketKeys = { 0, 9.223372036854776E18, 1.8446744073709552E19 }; + int i = 0; + for (Histogram.Bucket bucket : histo.getBuckets()) { + assertEquals(expectedBucketDocCounts[i], bucket.getDocCount()); + assertEquals(expectedBucketKeys[i], bucket.getKey()); + i++; + } + } + + // range agg + { + SearchResponse response = client().prepareSearch("idx") + .setSize(0) + .addAggregation( + range("ul_range").field("ul_field") + .addUnboundedTo(9.223372036854776E18) + .addRange(9.223372036854776E18, 1.8446744073709552E19) + .addUnboundedFrom(1.8446744073709552E19) + ) + .get(); + assertSearchResponse(response); + Range range = response.getAggregations().get("ul_range"); + + long[] expectedBucketDocCounts = { 3, 3, 4 }; + String[] expectedBucketKeys = { + "*-9.223372036854776E18", + "9.223372036854776E18-1.8446744073709552E19", + "1.8446744073709552E19-*" }; + int i = 0; + for (Range.Bucket bucket : range.getBuckets()) { + assertEquals(expectedBucketDocCounts[i], bucket.getDocCount()); + assertEquals(expectedBucketKeys[i], bucket.getKey()); + i++; + } + } + + // sum agg + { + SearchResponse response = client().prepareSearch("idx").setSize(0).addAggregation(sum("ul_sum").field("ul_field")).get(); + assertSearchResponse(response); + Sum sum = response.getAggregations().get("ul_sum"); + double expectedSum = Arrays.stream(values).mapToDouble(Number::doubleValue).sum(); + assertEquals(expectedSum, sum.getValue(), 0.001); + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml index 98bd40a0ca1ad..0341ae0cdc4db 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml @@ -147,15 +147,46 @@ setup: - match: { "hits.total.value": 5 } - match: {hits.hits.0._id: "1" } - - match: {hits.hits.0.sort: [0.0] } - # as sort is based on double representation, there is some loss of precision during converting longs to doubles - # thus both 9223372036854775807 and 9223372036854775808 are converted to 9.223372036854776E18 - # both 18446744073709551614 and 18446744073709551615 are converted to 1.8446744073709552E19 - # hence, we can't assert ids for the following sort results: - - match: {hits.hits.1.sort: [9.223372036854776E18] } # could be docs with _id 2 or 3 - - match: {hits.hits.2.sort: [9.223372036854776E18] } # could be docs with _id 2 or 3 - - match: {hits.hits.3.sort: [1.8446744073709552E19] } # could be docs with _id 4 or 5 - - match: {hits.hits.4.sort: [1.8446744073709552E19] } # could be docs with _id 4 or 5 + - match: {hits.hits.0.sort: [0] } + - match: {hits.hits.1._id: "2" } + - match: {hits.hits.1.sort: [9223372036854775807] } + - match: {hits.hits.2._id: "3" } + - match: {hits.hits.2.sort: [9223372036854775808] } + - match: {hits.hits.3._id: "4" } + - match: {hits.hits.3.sort: [18446744073709551614] } + - match: {hits.hits.4._id: "5" } + - match: {hits.hits.4.sort: [18446744073709551615] } + + - do: + search: + index: test1 + body: + sort: [ { ul: asc } ] + search_after: [9223372036854775808] + + - length: { hits.hits: 2 } + - match: {hits.hits.0._id: "4" } + - match: {hits.hits.0.sort: [18446744073709551614] } + - match: {hits.hits.1._id: "5" } + - match: {hits.hits.1.sort: [18446744073709551615] } + + - do: + search: + index: test1 + body: + sort: [ { ul: desc } ] + + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "5" } + - match: {hits.hits.0.sort: [18446744073709551615] } + - match: {hits.hits.1._id: "4" } + - match: {hits.hits.1.sort: [18446744073709551614] } + - match: {hits.hits.2._id: "3" } + - match: {hits.hits.2.sort: [9223372036854775808] } + - match: {hits.hits.3._id: "2" } + - match: {hits.hits.3.sort: [9223372036854775807] } + - match: {hits.hits.4._id: "1" } + - match: {hits.hits.4.sort: [0] } --- "Aggs": @@ -169,10 +200,12 @@ setup: ul_terms: terms: field: ul - - length: { aggregations.ul_terms.buckets: 3 } - - match: { aggregations.ul_terms.buckets.0.key: 9.223372036854776E18 } - - match: { aggregations.ul_terms.buckets.1.key: 1.8446744073709552E19 } - - match: { aggregations.ul_terms.buckets.2.key: 0.0 } + - length: { aggregations.ul_terms.buckets: 5 } + - match: { aggregations.ul_terms.buckets.0.key: 0 } + - match: { aggregations.ul_terms.buckets.1.key: 9223372036854775807 } + - match: { aggregations.ul_terms.buckets.2.key: 9223372036854775808 } + - match: { aggregations.ul_terms.buckets.3.key: 18446744073709551614 } + - match: { aggregations.ul_terms.buckets.4.key: 18446744073709551615 } - do: search: @@ -183,11 +216,14 @@ setup: ul_histogram: histogram: field: ul - interval: 9223372036854775808 + interval: 9223372036854775807 - length: { aggregations.ul_histogram.buckets: 3 } - match: { aggregations.ul_histogram.buckets.0.key: 0.0 } + - match: { aggregations.ul_histogram.buckets.0.doc_count: 1 } - match: { aggregations.ul_histogram.buckets.1.key: 9.223372036854776E18 } + - match: { aggregations.ul_histogram.buckets.1.doc_count: 2 } - match: { aggregations.ul_histogram.buckets.2.key: 1.8446744073709552E19 } + - match: { aggregations.ul_histogram.buckets.2.doc_count: 2 } - do: search: @@ -199,9 +235,11 @@ setup: range: field: ul ranges: [ - { "to": 9223372036854775807 }, - { "from": 9223372036854775807} + { "from": null, "to": 9223372036854775807 }, + { "from": 9223372036854775807, "to" : 18446744073709551614}, + { "from": 18446744073709551614} ] - - length: { aggregations.ul_range.buckets: 2 } + - length: { aggregations.ul_range.buckets: 3 } - match: { aggregations.ul_range.buckets.0.doc_count: 1 } - - match: { aggregations.ul_range.buckets.1.doc_count: 4 } + - match: { aggregations.ul_range.buckets.1.doc_count: 2 } + - match: { aggregations.ul_range.buckets.2.doc_count: 2 } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml index cd30440d24ab7..e9ecefbd49cde 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml @@ -21,13 +21,13 @@ body: | { "index": {"_id" : "1"} } { "ul": 0 } - { "index": {"_id" : "2"} } + { "index": {"_id" : "2_null"} } { "ul": null } - { "index": {"_id" : "3"} } + { "index": {"_id" : "3_null"} } { "ul": ""} { "index": {"_id" : "4"} } - { "ul": 18446744073709551615 } - { "index": {"_id" : "5"} } + { "ul": 18446744073709551614 } + { "index": {"_id" : "5_missing"} } {} # term query @@ -39,8 +39,8 @@ term: ul: 17446744073709551615 - match: { "hits.total.value": 2 } - - match: {hits.hits.0._id: "2" } - - match: {hits.hits.1._id: "3" } + - match: {hits.hits.0._id: "2_null" } + - match: {hits.hits.1._id: "3_null" } # asc sort @@ -51,14 +51,15 @@ sort: { ul : { order: asc, missing : "_last" } } - match: { "hits.total.value": 5 } - match: {hits.hits.0._id: "1" } - - match: {hits.hits.0.sort: [0.0] } - - match: {hits.hits.1._id: "2" } - - match: {hits.hits.1.sort: [1.7446744073709552E19] } - - match: {hits.hits.2._id: "3" } - - match: {hits.hits.2.sort: [1.7446744073709552E19] } + - match: {hits.hits.0.sort: [0] } + - match: {hits.hits.1._id: "2_null" } + - match: {hits.hits.1.sort: [17446744073709551615] } + - match: {hits.hits.2._id: "3_null" } + - match: {hits.hits.2.sort: [17446744073709551615] } - match: {hits.hits.3._id: "4" } - - match: {hits.hits.3.sort: [1.8446744073709552E19] } - - match: {hits.hits.4._id: "5" } + - match: {hits.hits.3.sort: [18446744073709551614] } + - match: {hits.hits.4._id: "5_missing" } + - match: {hits.hits.4.sort: [18446744073709551615] } # desc sort - do: @@ -67,13 +68,13 @@ body: sort: { ul: { order: desc, missing: "_first" } } - match: { "hits.total.value": 5 } - - match: {hits.hits.0._id: "5" } + - match: {hits.hits.0._id: "5_missing" } + - match: {hits.hits.0.sort: [18446744073709551615] } - match: {hits.hits.1._id: "4" } - - match: {hits.hits.1.sort: [1.8446744073709552E19] } - - match: {hits.hits.2._id: "2" } - - match: {hits.hits.2.sort: [1.7446744073709552E19] } - - match: {hits.hits.3._id: "3" } - - match: {hits.hits.3.sort: [1.7446744073709552E19] } + - match: {hits.hits.1.sort: [18446744073709551614] } + - match: {hits.hits.2._id: "2_null" } + - match: {hits.hits.2.sort: [17446744073709551615] } + - match: {hits.hits.3._id: "3_null" } + - match: {hits.hits.3.sort: [17446744073709551615] } - match: {hits.hits.4._id: "1" } - - match: {hits.hits.4.sort: [0.0] } - + - match: {hits.hits.4.sort: [0] } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml index ba37f57ee886c..080c7a155721d 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml @@ -1,21 +1,11 @@ ---- -"Different numeric types": +setup: - skip: version: " - 7.99.99" reason: "unsigned_long was added in 8.0" - do: indices.create: - index: test1 - body: - mappings: - properties: - my_counter: - type: double - - - do: - indices.create: - index: test2 + index: test_longs body: mappings: properties: @@ -24,7 +14,7 @@ - do: indices.create: - index: test3 + index: test_unsigned_longs body: mappings: properties: @@ -33,18 +23,7 @@ - do: bulk: - index: test1 - refresh: true - body: | - { "index": {"_id" : "1"} } - { "my_counter": 0 } - { "index": {"_id" : "2"} } - { "my_counter": 1000000 } - { "index": {"_id" : "3"} } - { "my_counter": 9223372036854775807 } - - do: - bulk: - index: test2 + index: test_longs refresh: true body: | { "index": {"_id" : "1"} } @@ -56,7 +35,7 @@ - do: bulk: - index: test3 + index: test_unsigned_longs refresh: true body: | { "index": {"_id" : "1"} } @@ -64,9 +43,13 @@ { "index": {"_id" : "2"} } { "my_counter": 1000000 } { "index": {"_id" : "3"} } + { "my_counter": 9223372036854775807 } + { "index": {"_id" : "4"} } { "my_counter": 18446744073709551615 } +--- +"Querying of different numeric types is supported": - do: search: index: test* @@ -76,7 +59,7 @@ range: my_counter: gte: 0 - - match: { "hits.total.value": 9 } + - match: { "hits.total.value": 7 } - do: search: @@ -88,7 +71,7 @@ my_counter: gt: 0 lt: 9223372036854775807 - - match: { "hits.total.value": 3 } + - match: { "hits.total.value": 2 } - do: search: @@ -100,3 +83,35 @@ my_counter: gte: 9223372036854775807 - match: { "hits.total.value": 3 } + + +--- +"Aggregation of different numeric types is supported": + - do: + search: + index: test* + body: + size: 0 + aggs: + my_counter_sum: + sum: + field: my_counter + - match: { aggregations.my_counter_sum.value: 3.68934881474211E19 } + + - do: + search: + index: test* + body: + size: 0 + aggs: + my_counter_histo: + histogram: + field: my_counter + interval: 9223372036854775807 + - length: { aggregations.my_counter_histo.buckets: 3 } + - match: { aggregations.my_counter_histo.buckets.0.key: 0.0 } + - match: { aggregations.my_counter_histo.buckets.0.doc_count: 4 } + - match: { aggregations.my_counter_histo.buckets.1.key: 9.223372036854776E18 } + - match: { aggregations.my_counter_histo.buckets.1.doc_count: 2 } + - match: { aggregations.my_counter_histo.buckets.2.key: 1.8446744073709552E19 } + - match: { aggregations.my_counter_histo.buckets.2.doc_count: 1 } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/50_script_values.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/50_script_values.yml new file mode 100644 index 0000000000000..7955cc7455178 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/50_script_values.yml @@ -0,0 +1,110 @@ +setup: + + - skip: + version: " - 7.99.99" + reason: "unsigned_long was added in 8.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + ul: + type: unsigned_long + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "ul": 0 } + { "index": {"_id" : "2"} } + { "ul": 9223372036854775807 } + { "index": {"_id" : "3"} } + { "ul": 9223372036854775808 } + { "index": {"_id" : "4"} } + { "ul": 18446744073709551614 } + { "index": {"_id" : "5"} } + { "ul": 18446744073709551615 } + +--- +"Scripted fields values return BigInteger or Long": + - do: + search: + index: test1 + body: + sort: [ { ul: desc } ] + script_fields: + scripted_ul: + script: + source: "doc['ul'].value" + + - match: { hits.hits.0.fields.scripted_ul.0: 18446744073709551615 } + - match: { hits.hits.1.fields.scripted_ul.0: 18446744073709551614 } + - match: { hits.hits.2.fields.scripted_ul.0: 9223372036854775808 } + - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 } + - match: { hits.hits.4.fields.scripted_ul.0: 0 } + +--- +"Scripted sort values": + - do: + search: + index: test1 + body: + sort: + _script: + order: desc + type: number + script: + source: "doc['ul'].value" + + - match: { hits.hits.0.sort: [1.8446744073709552E19] } + - match: { hits.hits.1.sort: [1.8446744073709552E19] } + - match: { hits.hits.2.sort: [9.223372036854776E18] } + - match: { hits.hits.3.sort: [9.223372036854776E18] } + - match: { hits.hits.4.sort: [0.0] } + +--- +"Script query": + - do: + search: + index: test1 + body: + query: + bool: + filter: + script: + script: + source: "doc['ul'].value.doubleValue() > 10E18" + - match: { hits.total.value: 2 } + - match: { hits.hits.0._id: "4" } + - match: { hits.hits.1._id: "5" } + + - do: + search: + index: test1 + body: + size: 0 + query: + bool: + filter: + script: + script: + source: "doc['ul'].size() > 0" + - match: { hits.total.value: 5 } + +--- +"script_score query": + - do: + search: + index: test1 + body: + query: + script_score: + query: {match_all: {}} + script: + source: "doc['ul'].value" + + - match: { hits.total.value: 5 } From ada3422c7f73a22c714974c1542a4e84b3ecadee Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 18 Aug 2020 18:06:12 -0400 Subject: [PATCH 03/13] Modifications after master merge --- .../unsignedlong/UnsignedLongFieldMapper.java | 37 +++++++++++++++++-- .../UnsignedLongIndexFieldData.java | 5 --- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 4d083664a12c6..babd875d7433f 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -378,7 +378,8 @@ protected static Long parseUpperRangeTerm(Object value, boolean include) { private Explicit ignoreMalformed; private final String nullValue; - private final Long nullValueNumeric; + private final Long nullValueIndexed; // null value to use for indexing, represented as shifted to signed long range + private final Number nullValueFormatted; // null value to use in place of a {@code null} value in the document source private UnsignedLongFieldMapper( String simpleName, @@ -391,7 +392,15 @@ private UnsignedLongFieldMapper( ) { super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); this.nullValue = nullValue; - this.nullValueNumeric = nullValue == null ? null : convertToSignedLong(parseUnsignedLong(nullValue)); + if (nullValue == null) { + this.nullValueIndexed = null; + this.nullValueFormatted = null; + } else { + long parsed = parseUnsignedLong(nullValue); + this.nullValueIndexed = convertToSignedLong(parsed); + this.nullValueFormatted = parsed >= 0 ? parsed : BigInteger.valueOf(parsed).and(BIGINTEGER_2_64_MINUS_ONE); + } + this.ignoreMalformed = ignoreMalformed; } @@ -410,6 +419,11 @@ protected UnsignedLongFieldMapper clone() { return (UnsignedLongFieldMapper) super.clone(); } + @Override + protected Number nullValue() { + return nullValueFormatted; + } + @Override protected void parseCreateField(ParseContext context) throws IOException { XContentParser parser = context.parser(); @@ -437,7 +451,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } } if (numericValue == null) { - numericValue = nullValueNumeric; + numericValue = nullValueIndexed; if (numericValue == null) return; } else { numericValue = convertToSignedLong(numericValue); @@ -454,6 +468,23 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected Number parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + if (value.equals("")) { + return nullValueFormatted; + } + long ulValue = parseUnsignedLong(value); + if (ulValue >= 0) { + return ulValue; + } else { + return BigInteger.valueOf(ulValue).and(BIGINTEGER_2_64_MINUS_ONE); + } + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { UnsignedLongFieldMapper mergeWith = (UnsignedLongFieldMapper) other; diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java index 8322aeb2c7da2..eece0ff52daa4 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java @@ -43,11 +43,6 @@ protected boolean sortRequiresCustomComparator() { return true; } - @Override - public void clear() { - signedLongIFD.clear(); - } - @Override public NumericType getNumericType() { return NumericType.LONG; From e903940c82c2e9e494465f5a1e8633b1bd834d53 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 19 Aug 2020 16:29:29 -0400 Subject: [PATCH 04/13] Rename methods --- .../unsignedlong/UnsignedLongFieldMapper.java | 16 ++++++++-------- .../unsignedlong/UnsignedLongLeafFieldData.java | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index babd875d7433f..b332d1660a1bf 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -171,7 +171,7 @@ public Query termQuery(Object value, QueryShardContext context) { if (longValue == null) { return new MatchNoDocsQuery(); } - Query query = LongPoint.newExactQuery(name(), convertToSignedLong(longValue)); + Query query = LongPoint.newExactQuery(name(), unsignedToSortableSignedLong(longValue)); if (boost() != 1f) { query = new BoostQuery(query, boost()); } @@ -187,7 +187,7 @@ public Query termsQuery(List values, QueryShardContext context) { Object value = values.get(i); Long longValue = parseTerm(value); if (longValue != null) { - lvalues[upTo++] = convertToSignedLong(longValue); + lvalues[upTo++] = unsignedToSortableSignedLong(longValue); } } if (upTo == 0) { @@ -211,12 +211,12 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower if (lowerTerm != null) { Long lt = parseLowerRangeTerm(lowerTerm, includeLower); if (lt == null) return new MatchNoDocsQuery(); - l = convertToSignedLong(lt); + l = unsignedToSortableSignedLong(lt); } if (upperTerm != null) { Long ut = parseUpperRangeTerm(upperTerm, includeUpper); if (ut == null) return new MatchNoDocsQuery(); - u = convertToSignedLong(ut); + u = unsignedToSortableSignedLong(ut); } if (l > u) return new MatchNoDocsQuery(); @@ -397,7 +397,7 @@ private UnsignedLongFieldMapper( this.nullValueFormatted = null; } else { long parsed = parseUnsignedLong(nullValue); - this.nullValueIndexed = convertToSignedLong(parsed); + this.nullValueIndexed = unsignedToSortableSignedLong(parsed); this.nullValueFormatted = parsed >= 0 ? parsed : BigInteger.valueOf(parsed).and(BIGINTEGER_2_64_MINUS_ONE); } @@ -454,7 +454,7 @@ protected void parseCreateField(ParseContext context) throws IOException { numericValue = nullValueIndexed; if (numericValue == null) return; } else { - numericValue = convertToSignedLong(numericValue); + numericValue = unsignedToSortableSignedLong(numericValue); } boolean docValued = fieldType().hasDocValues(); @@ -541,7 +541,7 @@ private static long parseUnsignedLong(Object value) { * @param value – unsigned long value in the range [0; 2^64-1], values greater than 2^63-1 are negative * @return signed long value in the range [-2^63; 2^63-1] */ - private static long convertToSignedLong(long value) { + private static long unsignedToSortableSignedLong(long value) { // subtracting 2^63 or 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 // equivalent to flipping the first bit return value ^ MASK_2_63; @@ -552,7 +552,7 @@ private static long convertToSignedLong(long value) { * @param value – signed long value in the range [-2^63; 2^63-1] * @return unsigned long value in the range [0; 2^64-1], values greater then 2^63-1 are negative */ - protected static long convertToOriginal(long value) { + protected static long sortableSignedLongToUnsigned(long value) { // adding 2^63 or 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 // equivalent to flipping the first bit return value ^ MASK_2_63; diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java index ef6c5643d88b8..8a872e0f9157f 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -18,7 +18,7 @@ import java.io.IOException; -import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.convertToOriginal; +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.sortableSignedLongToUnsigned; public class UnsignedLongLeafFieldData implements LeafNumericFieldData { private final LeafNumericFieldData signedLongFD; @@ -91,7 +91,7 @@ public void close() { private static double convertUnsignedLongToDouble(long value) { if (value < 0L) { - return convertToOriginal(value); // add 2 ^ 63 + return sortableSignedLongToUnsigned(value); // add 2 ^ 63 } else { // add 2 ^ 63 as a double to make sure there is no overflow and final result is positive return 0x1.0p63 + value; From 9e057c08334d30f489903b452ff7d81142541866 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 8 Sep 2020 16:00:04 -0400 Subject: [PATCH 05/13] Address Jim's feedback --- .../mapping/types/unsigned_long.asciidoc | 20 +++++++++-------- .../common/io/stream/StreamOutput.java | 12 +++++----- .../bucket/terms/DoubleTerms.java | 3 ++- .../aggregations/bucket/terms/LongTerms.java | 21 ++++++++++++++++++ .../unsigned_long/40_different_numeric.yml | 22 +++++++++++++++++++ 5 files changed, 62 insertions(+), 16 deletions(-) diff --git a/docs/reference/mapping/types/unsigned_long.asciidoc b/docs/reference/mapping/types/unsigned_long.asciidoc index 6863847ba97af..c1c6d01a23cdd 100644 --- a/docs/reference/mapping/types/unsigned_long.asciidoc +++ b/docs/reference/mapping/types/unsigned_long.asciidoc @@ -86,7 +86,7 @@ GET /my_index/_search For queries with sort on an `unsigned_long` field, for a particular document {es} returns a sort value of the type `Long` if the value of this document is within the range of long values, -or of the type `BigIntger` if the value exceeds this range. +or of the type `BigInteger` if the value exceeds this range. WARNING: Not all {es} clients can properly handle big integer values. @@ -104,17 +104,19 @@ GET /my_index/_search Similarly to sort values, script values of an `unsigned_long` field produce `BigInteger` or `Long` values. The same values: `BigInteger` or -`Long` are returned as keys for `terms` aggregation. +`Long` are used for `terms` aggregation. ==== Queries with mixed numeric types -Search queries across several numeric types one of which `unsigned_long` are supported, -except queries with sort. Thus, a sort query across two indexes where the same field -is `unsigned_long` in one index, and `long` in another, doesn't produce correct results -and must be avoided. If there is a need for a such kind of sorting, script based -sorting can be used instead. -Aggregations across several numeric types one of which `unsigned_long` are supported, -except a terms aggregation. +Search queries across several numeric types one of which `unsigned_long` are +supported, except queries with sort. Thus, a sort query across two indexes +where the same field name has an `unsigned_long` type in one index, +and `long` type in another, doesn't produce correct results and must +be avoided. If there is a need for such kind of sorting, script based sorting +can be used instead. + +Aggregations across several numeric types one of which is `unsigned_long` are +supported. In this case, values are converted to the `double` type. [[unsigned-long-params]] diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 0c88ee22b7392..bb72b4ec61901 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -829,12 +829,12 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep o.writeCollection((Set) v, StreamOutput::writeGenericValue); }), entry( - // TODO: improve serialization of BigInteger - BigInteger.class, - (o, v) -> { - o.writeByte((byte) 26); - o.writeString(v.toString()); - } + // TODO: improve serialization of BigInteger + BigInteger.class, + (o, v) -> { + o.writeByte((byte) 26); + o.writeString(v.toString()); + } )); private static Class getGenericType(Object value) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java index cf0a064236316..9147f6a7ad19d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java @@ -146,7 +146,8 @@ protected Bucket[] createBucketsArray(int size) { public InternalAggregation reduce(List aggregations, ReduceContext reduceContext) { boolean promoteToDouble = false; for (InternalAggregation agg : aggregations) { - if (agg instanceof LongTerms && ((LongTerms) agg).format == DocValueFormat.RAW) { + if (agg instanceof LongTerms && + (((LongTerms) agg).format == DocValueFormat.RAW || ((LongTerms) agg).format == DocValueFormat.UNSIGNED_LONG_SHIFTED) ) { /* * this terms agg mixes longs and doubles, we must promote longs to doubles to make the internal aggs * compatible diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java index de606ad0de62f..8221867c8d478 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java @@ -156,10 +156,31 @@ protected Bucket[] createBucketsArray(int size) { @Override public InternalAggregation reduce(List aggregations, ReduceContext reduceContext) { + boolean unsignedLongFormat = false; + boolean rawFormat = false; for (InternalAggregation agg : aggregations) { if (agg instanceof DoubleTerms) { return agg.reduce(aggregations, reduceContext); } + if (agg instanceof LongTerms) { + if (((LongTerms) agg).format == DocValueFormat.RAW) { + rawFormat = true; + } else if (((LongTerms) agg).format == DocValueFormat.UNSIGNED_LONG_SHIFTED) { + unsignedLongFormat = true; + } + } + } + if (rawFormat && unsignedLongFormat) { // if we have mixed formats, convert results to double format + List newAggs = new ArrayList<>(aggregations.size()); + for (InternalAggregation agg : aggregations) { + if (agg instanceof LongTerms) { + DoubleTerms dTerms = LongTerms.convertLongTermsToDouble((LongTerms) agg, format); + newAggs.add(dTerms); + } else { + newAggs.add(agg); + } + } + return newAggs.get(0).reduce(newAggs, reduceContext); } return super.reduce(aggregations, reduceContext); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml index 080c7a155721d..5ffe49f1ac8fb 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml @@ -87,6 +87,7 @@ setup: --- "Aggregation of different numeric types is supported": + # sum agg - do: search: index: test* @@ -98,6 +99,7 @@ setup: field: my_counter - match: { aggregations.my_counter_sum.value: 3.68934881474211E19 } + # histogram agg - do: search: index: test* @@ -115,3 +117,23 @@ setup: - match: { aggregations.my_counter_histo.buckets.1.doc_count: 2 } - match: { aggregations.my_counter_histo.buckets.2.key: 1.8446744073709552E19 } - match: { aggregations.my_counter_histo.buckets.2.doc_count: 1 } + + # terms agg bucket values are converted to double + - do: + search: + index: test* + body: + size: 0 + aggs: + my_counter_terms: + terms: + field: my_counter + - length: { aggregations.my_counter_terms.buckets: 4 } + - match: { aggregations.my_counter_terms.buckets.0.key: 0.0 } + - match: { aggregations.my_counter_terms.buckets.0.doc_count: 2 } + - match: { aggregations.my_counter_terms.buckets.1.key: 1000000.0 } + - match: { aggregations.my_counter_terms.buckets.1.doc_count: 2 } + - match: { aggregations.my_counter_terms.buckets.2.key: 9.223372036854776E18 } + - match: { aggregations.my_counter_terms.buckets.2.doc_count: 2 } + - match: { aggregations.my_counter_terms.buckets.3.key: 1.8446744073709552E19 } + - match: { aggregations.my_counter_terms.buckets.3.doc_count: 1 } From ab54a23fee318cc4827c2e8446949fc01035d3ad Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 8 Sep 2020 16:28:04 -0400 Subject: [PATCH 06/13] Include unsigned_long docs into numeric type --- docs/reference/mapping/types.asciidoc | 4 +- docs/reference/mapping/types/numeric.asciidoc | 4 +- .../mapping/types/unsigned_long.asciidoc | 43 ------------------- 3 files changed, 4 insertions(+), 47 deletions(-) diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index d610404710f3a..8f65257ffddd8 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -9,7 +9,7 @@ document: === Core data types string:: <>, <> and <> -<>:: `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`, <> +<>:: `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`, `unsigned_long` <>:: `date` <>:: `date_nanos` <>:: `boolean` @@ -137,5 +137,3 @@ include::types/shape.asciidoc[] include::types/constant-keyword.asciidoc[] include::types/wildcard.asciidoc[] - -include::types/unsigned_long.asciidoc[] diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index 0bc9c64055c06..d37d213ab6ee2 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -15,7 +15,7 @@ The following numeric types are supported: `float`:: A single-precision 32-bit IEEE 754 floating point number, restricted to finite values. `half_float`:: A half-precision 16-bit IEEE 754 floating point number, restricted to finite values. `scaled_float`:: A floating point number that is backed by a `long`, scaled by a fixed `double` scaling factor. -`unsigned_long`:: An <> with a minimum value of 0 and a maximum value of +2^64^-1+. +`unsigned_long`:: An unsigned 64-bit integer with a minimum value of 0 and a maximum value of +2^64^-1+. Below is an example of configuring a mapping with numeric fields: @@ -165,3 +165,5 @@ The following parameters are accepted by numeric types: sorting) will behave as if the document had a value of +2.3+. High values of `scaling_factor` improve accuracy but also increase space requirements. This parameter is required. + +include::unsigned_long.asciidoc[] diff --git a/docs/reference/mapping/types/unsigned_long.asciidoc b/docs/reference/mapping/types/unsigned_long.asciidoc index c1c6d01a23cdd..75debcd34f01c 100644 --- a/docs/reference/mapping/types/unsigned_long.asciidoc +++ b/docs/reference/mapping/types/unsigned_long.asciidoc @@ -3,10 +3,6 @@ [[unsigned-long]] === Unsigned long data type -++++ -Unsigned long -++++ - Unsigned long is a numeric field type that represents an unsigned 64-bit integer with a minimum value of 0 and a maximum value of +2^64^-1+ (from 0 to 18446744073709551615 inclusive). @@ -117,42 +113,3 @@ can be used instead. Aggregations across several numeric types one of which is `unsigned_long` are supported. In this case, values are converted to the `double` type. - - -[[unsigned-long-params]] -==== Parameters for unsigned long fields - -The following parameters are accepted: - -[horizontal] - -<>:: - - Should the field be stored on disk in a column-stride fashion, so that it - can later be used for sorting, aggregations, or scripting? Accepts `true` - (default) or `false`. - -<>:: - - If `true`, malformed numbers are ignored. If `false` (default), malformed - numbers throw an exception and reject the whole document. - -<>:: - - Should the field be searchable? Accepts `true` (default) and `false`. - -<>:: - - Accepts a numeric value of the same `type` as the field which is - substituted for any explicit `null` values. Defaults to `null`, which - means the field is treated as missing. - -<>:: - - Whether the field value should be stored and retrievable separately from - the <> field. Accepts `true` or `false` - (default). - -<>:: - - Metadata about the field. From 2b567c961f4dc9331ef7eef25c4c9c8b271a67c8 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 9 Sep 2020 16:02:59 -0400 Subject: [PATCH 07/13] Convert UnsignedLongFieldMapper to parametrized - Convert UnsignedLongFieldMapper to a parametrized form - Small adjustments in UnsignedLongScriptDocValues --- .../unsignedlong/UnsignedLongFieldMapper.java | 224 ++++++-------- .../UnsignedLongMapperPlugin.java | 5 +- .../UnsignedLongScriptDocValues.java | 6 +- .../UnsignedLongFieldMapperTests.java | 285 +++++------------- 4 files changed, 177 insertions(+), 343 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index b332d1660a1bf..8ae29396f2cce 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -8,11 +8,8 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.exc.InputCoercionException; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; @@ -23,23 +20,26 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; -import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SimpleMappedFieldType; +import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; -import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.math.BigDecimal; @@ -47,99 +47,85 @@ import java.time.ZoneId; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.function.Supplier; -public class UnsignedLongFieldMapper extends FieldMapper { - protected static long MASK_2_63 = 0x8000000000000000L; - private static BigInteger BIGINTEGER_2_64_MINUS_ONE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); // 2^64 -1 - private static BigDecimal BIGDECIMAL_2_64_MINUS_ONE = new BigDecimal(BIGINTEGER_2_64_MINUS_ONE); - +public class UnsignedLongFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "unsigned_long"; - // use the same default as numbers - private static final FieldType FIELD_TYPE = new FieldType(); - static { - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - } - - public static class Builder extends FieldMapper.Builder { - private Boolean ignoreMalformed; - private String nullValue; - - public Builder(String name) { - super(name, FIELD_TYPE); - builder = this; - } + private static final long MASK_2_63 = 0x8000000000000000L; + static final BigInteger BIGINTEGER_2_64_MINUS_ONE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); // 2^64 -1 + private static final BigDecimal BIGDECIMAL_2_64_MINUS_ONE = new BigDecimal(BIGINTEGER_2_64_MINUS_ONE); - public Builder ignoreMalformed(boolean ignoreMalformed) { - this.ignoreMalformed = ignoreMalformed; - return builder; - } + private static UnsignedLongFieldMapper toType(FieldMapper in) { + return (UnsignedLongFieldMapper) in; + } - @Override - public Builder indexOptions(IndexOptions indexOptions) { - throw new MapperParsingException("index_options not allowed in field [" + name + "] of type [" + CONTENT_TYPE + "]"); + public static class Builder extends ParametrizedFieldMapper.Builder { + private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); + private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); + private final Parameter> ignoreMalformed; + private final Parameter nullValue; + private final Parameter> meta = Parameter.metaParam(); + + public Builder(String name, Settings settings) { + this(name, IGNORE_MALFORMED_SETTING.get(settings)); + } + + private Builder(String name, boolean ignoreMalformedByDefault) { + super(name); + this.ignoreMalformed = Parameter.explicitBoolParam( + "ignore_malformed", + true, + m -> toType(m).ignoreMalformed, + ignoreMalformedByDefault + ); + this.nullValue = new Parameter<>( + "null_value", + false, + () -> null, + (n, c, o) -> parseNullValueAsString(o), + m -> toType(m).nullValue + ).acceptsNull(); } - protected Explicit ignoreMalformed(BuilderContext context) { - if (ignoreMalformed != null) { - return new Explicit<>(ignoreMalformed, true); - } - if (context.indexSettings() != null) { - return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); + private String parseNullValueAsString(Object o) { + if (o == null) return null; + try { + parseUnsignedLong(o); // confirm that null_value is a proper unsigned_long + return (o instanceof BytesRef) ? ((BytesRef) o).utf8ToString() : o.toString(); + } catch (Exception e) { + throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e); } - return NumberFieldMapper.Defaults.IGNORE_MALFORMED; } - public Builder nullValue(String nullValue) { - this.nullValue = nullValue; + Builder nullValue(String nullValue) { + this.nullValue.setValue(nullValue); return this; } @Override - public UnsignedLongFieldMapper build(BuilderContext context) { - UnsignedLongFieldType type = new UnsignedLongFieldType(buildFullName(context), indexed, hasDocValues, meta); - return new UnsignedLongFieldMapper( - name, - fieldType, - type, - ignoreMalformed(context), - multiFieldsBuilder.build(this, context), - copyTo, - nullValue - ); + protected List> getParameters() { + return List.of(indexed, hasDocValues, stored, ignoreMalformed, nullValue, meta); } - } - public static class TypeParser implements Mapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = new Builder(name); - TypeParsers.parseField(builder, name, node, parserContext); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String propName = entry.getKey(); - Object propNode = entry.getValue(); - if (propName.equals("null_value")) { - if (propNode == null) { - throw new MapperParsingException("Property [null_value] cannot be null."); - } - parseUnsignedLong(propNode); // confirm that null_value is a proper unsigned_long - String nullValue = (propNode instanceof BytesRef) ? ((BytesRef) propNode).utf8ToString() : propNode.toString(); - builder.nullValue(nullValue); - iterator.remove(); - } else if (propName.equals("ignore_malformed")) { - builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode, name + ".ignore_malformed")); - iterator.remove(); - } - } - return builder; + public UnsignedLongFieldMapper build(BuilderContext context) { + UnsignedLongFieldType fieldType = new UnsignedLongFieldType( + buildFullName(context), + indexed.getValue(), + hasDocValues.getValue(), + meta.getValue() + ); + return new UnsignedLongFieldMapper(name, fieldType, multiFieldsBuilder.build(this, context), copyTo.build(), this); } } + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings())); + public static final class UnsignedLongFieldType extends SimpleMappedFieldType { public UnsignedLongFieldType(String name, boolean indexed, boolean hasDocValues, Map meta) { @@ -235,7 +221,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return (cache, breakerService, mapperService) -> { final IndexNumericFieldData signedLongValues = new SortedNumericIndexFieldData.Builder( @@ -376,22 +362,29 @@ protected static Long parseUpperRangeTerm(Object value, boolean include) { } } - private Explicit ignoreMalformed; + private final boolean indexed; + private final boolean hasDocValues; + private final boolean stored; + private final Explicit ignoreMalformed; + private final boolean ignoreMalformedByDefault; private final String nullValue; private final Long nullValueIndexed; // null value to use for indexing, represented as shifted to signed long range private final Number nullValueFormatted; // null value to use in place of a {@code null} value in the document source private UnsignedLongFieldMapper( String simpleName, - FieldType fieldType, - UnsignedLongFieldType mappedFieldType, - Explicit ignoreMalformed, + MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, - String nullValue + Builder builder ) { - super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); - this.nullValue = nullValue; + super(simpleName, mappedFieldType, multiFields, copyTo); + this.indexed = builder.indexed.getValue(); + this.hasDocValues = builder.hasDocValues.getValue(); + this.stored = builder.stored.getValue(); + this.ignoreMalformed = builder.ignoreMalformed.getValue(); + this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value(); + this.nullValue = builder.nullValue.getValue(); if (nullValue == null) { this.nullValueIndexed = null; this.nullValueFormatted = null; @@ -400,8 +393,6 @@ private UnsignedLongFieldMapper( this.nullValueIndexed = unsignedToSortableSignedLong(parsed); this.nullValueFormatted = parsed >= 0 ? parsed : BigInteger.valueOf(parsed).and(BIGINTEGER_2_64_MINUS_ONE); } - - this.ignoreMalformed = ignoreMalformed; } @Override @@ -419,11 +410,6 @@ protected UnsignedLongFieldMapper clone() { return (UnsignedLongFieldMapper) super.clone(); } - @Override - protected Number nullValue() { - return nullValueFormatted; - } - @Override protected void parseCreateField(ParseContext context) throws IOException { XContentParser parser = context.parser(); @@ -457,52 +443,38 @@ protected void parseCreateField(ParseContext context) throws IOException { numericValue = unsignedToSortableSignedLong(numericValue); } - boolean docValued = fieldType().hasDocValues(); - boolean indexed = fieldType().isSearchable(); - boolean stored = fieldType.stored(); - - List fields = NumberFieldMapper.NumberType.LONG.createFields(fieldType().name(), numericValue, indexed, docValued, stored); - context.doc().addAll(fields); - if (docValued == false && (indexed || stored)) { + context.doc() + .addAll(NumberFieldMapper.NumberType.LONG.createFields(fieldType().name(), numericValue, indexed, hasDocValues, stored)); + if (hasDocValues == false && (stored || indexed)) { createFieldNamesField(context); } } @Override - protected Number parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - if (value.equals("")) { - return nullValueFormatted; - } - long ulValue = parseUnsignedLong(value); - if (ulValue >= 0) { - return ulValue; - } else { - return BigInteger.valueOf(ulValue).and(BIGINTEGER_2_64_MINUS_ONE); - } - } - - @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - UnsignedLongFieldMapper mergeWith = (UnsignedLongFieldMapper) other; - if (mergeWith.ignoreMalformed.explicit()) { - this.ignoreMalformed = mergeWith.ignoreMalformed; - } + return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValueFormatted) { + @Override + protected Object parseSourceValue(Object value) { + if (value.equals("")) { + return nullValueFormatted; + } + long ulValue = parseUnsignedLong(value); + if (ulValue >= 0) { + return ulValue; + } else { + return BigInteger.valueOf(ulValue).and(BIGINTEGER_2_64_MINUS_ONE); + } + } + }; } @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - - if (includeDefaults || ignoreMalformed.explicit()) { - builder.field("ignore_malformed", ignoreMalformed.value()); - } - if (nullValue != null) { - builder.field("null_value", nullValue); - } + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName(), ignoreMalformedByDefault).init(this); } /** diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java index a3ea313403b53..85dd071f6e465 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.unsignedlong; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; @@ -17,11 +16,9 @@ public class UnsignedLongMapperPlugin extends Plugin implements MapperPlugin { - public UnsignedLongMapperPlugin(Settings settings) {} - @Override public Map getMappers() { - return singletonMap(UnsignedLongFieldMapper.CONTENT_TYPE, new UnsignedLongFieldMapper.TypeParser()); + return singletonMap(UnsignedLongFieldMapper.CONTENT_TYPE, UnsignedLongFieldMapper.PARSER); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java index 7b6e3329769a5..680033d078abf 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java @@ -15,7 +15,7 @@ public class UnsignedLongScriptDocValues extends ScriptDocValues { private final SortedNumericDocValues in; - private Number[] values = new Number[0]; + private long[] values = new long[0]; private int count; /** @@ -30,7 +30,7 @@ public void setNextDocId(int docId) throws IOException { if (in.advanceExact(docId)) { resize(in.docValueCount()); for (int i = 0; i < count; i++) { - values[i] = (Number) DocValueFormat.UNSIGNED_LONG_SHIFTED.format(in.nextValue()); + values[i] = in.nextValue(); } } else { resize(0); @@ -57,7 +57,7 @@ public Number get(int index) { "A document doesn't have a value for a field! Use doc[].size()==0 to check if a document is missing a field!" ); } - return values[index]; + return (Number) DocValueFormat.UNSIGNED_LONG_SHIFTED.format(values[index]); } @Override diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 1e32b7be7fee5..6905b90d03bc5 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -8,86 +8,61 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; -import org.elasticsearch.index.mapper.FieldMapperTestCase; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.junit.Before; import java.io.IOException; import java.math.BigInteger; import java.util.Collection; -import java.util.Set; +import java.util.List; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.BIGINTEGER_2_64_MINUS_ONE; import static org.hamcrest.Matchers.containsString; -public class UnsignedLongFieldMapperTests extends FieldMapperTestCase { - - IndexService indexService; - DocumentMapperParser parser; - - @Before - public void setup() { - indexService = createIndex("test"); - parser = indexService.mapperService().documentMapperParser(); - } - - @Override - protected Collection> getPlugins() { - return pluginList(UnsignedLongMapperPlugin.class, LocalStateCompositeXPackPlugin.class); - } +public class UnsignedLongFieldMapperTests extends MapperTestCase { @Override - protected Set unsupportedProperties() { - return Set.of("analyzer", "similarity"); + protected Collection getPlugins() { + return List.of(new UnsignedLongMapperPlugin()); } @Override - protected UnsignedLongFieldMapper.Builder newBuilder() { - return new UnsignedLongFieldMapper.Builder("my_unsigned_long"); + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "unsigned_long"); } public void testDefaults() throws Exception { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .endObject() - .endObject() - .endObject() - .endObject() - ); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); - assertEquals(mapping, mapper.mappingSource().toString()); + XContentBuilder mapping = fieldMapping(b -> b.field("type", "unsigned_long")); + DocumentMapper mapper = createDocumentMapper(mapping); + assertEquals(Strings.toString(mapping), mapper.mappingSource().toString()); - // test that indexing values as string + // test indexing of values as string { ParsedDocument doc = mapper.parse( new SourceToParse( "test", "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() - ), + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "18446744073709551615").endObject()), XContentType.JSON ) ); - IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(2, fields.length); IndexableField pointField = fields[0]; assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); @@ -105,13 +80,11 @@ public void testDefaults() throws Exception { new SourceToParse( "test", "2", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", 9223372036854775807L).endObject() - ), + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", 9223372036854775807L).endObject()), XContentType.JSON ) ); - IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(2, fields.length); IndexableField pointField = fields[0]; assertEquals(-1L, pointField.numericValue().longValue()); @@ -125,7 +98,7 @@ public void testDefaults() throws Exception { new SourceToParse( "test", "3", - BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", 10.5).endObject()), + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", 10.5).endObject()), XContentType.JSON ) ); @@ -135,33 +108,17 @@ public void testDefaults() throws Exception { } public void testNotIndexed() throws Exception { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .field("index", false) - .endObject() - .endObject() - .endObject() - .endObject() - ); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); - assertEquals(mapping, mapper.mappingSource().toString()); + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "unsigned_long").field("index", false))); ParsedDocument doc = mapper.parse( new SourceToParse( "test", "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() - ), + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "18446744073709551615").endObject()), XContentType.JSON ) ); - IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(1, fields.length); IndexableField dvField = fields[0]; assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); @@ -169,33 +126,17 @@ public void testNotIndexed() throws Exception { } public void testNoDocValues() throws Exception { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .field("doc_values", false) - .endObject() - .endObject() - .endObject() - .endObject() - ); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); - assertEquals(mapping, mapper.mappingSource().toString()); + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "unsigned_long").field("doc_values", false))); ParsedDocument doc = mapper.parse( new SourceToParse( "test", "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() - ), + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "18446744073709551615").endObject()), XContentType.JSON ) ); - IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(1, fields.length); IndexableField pointField = fields[0]; assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); @@ -203,33 +144,17 @@ public void testNoDocValues() throws Exception { } public void testStore() throws Exception { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .field("store", true) - .endObject() - .endObject() - .endObject() - .endObject() - ); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); - assertEquals(mapping, mapper.mappingSource().toString()); + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "unsigned_long").field("store", true))); ParsedDocument doc = mapper.parse( new SourceToParse( "test", "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() - ), + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "18446744073709551615").endObject()), XContentType.JSON ) ); - IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(3, fields.length); IndexableField pointField = fields[0]; assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); @@ -242,81 +167,46 @@ public void testStore() throws Exception { assertEquals(9223372036854775807L, storedField.numericValue().longValue()); } - public void testCoerceMappingParameterIsIllegal() throws Exception { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .field("coerce", false) - .endObject() - .endObject() - .endObject() - .endObject() + public void testCoerceMappingParameterIsIllegal() { + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> createMapperService(fieldMapping(b -> b.field("type", "unsigned_long").field("coerce", false))) + ); + assertThat( + e.getMessage(), + containsString("Failed to parse mapping: unknown parameter [coerce] on mapper [field] of type [unsigned_long]") ); - ThrowingRunnable runnable = () -> parser.parse("_doc", new CompressedXContent(mapping)); - MapperParsingException e = expectThrows(MapperParsingException.class, runnable); - assertEquals(e.getMessage(), "Mapping definition for [my_unsigned_long] has unsupported parameters: [coerce : false]"); } public void testNullValue() throws IOException { // test that if null value is not defined, field is not indexed { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .endObject() - .endObject() - .endObject() - .endObject() - ); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); - assertEquals(mapping, mapper.mappingSource().toString()); - + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); ParsedDocument doc = mapper.parse( new SourceToParse( "test", "1", - BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("my_unsigned_long").endObject()), + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("field").endObject()), XContentType.JSON ) ); - assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("my_unsigned_long")); + assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field")); } // test that if null value is defined, it is used { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .field("null_value", "18446744073709551615") - .endObject() - .endObject() - .endObject() - .endObject() + DocumentMapper mapper = createDocumentMapper( + fieldMapping(b -> b.field("type", "unsigned_long").field("null_value", "18446744073709551615")) ); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); - assertEquals(mapping, mapper.mappingSource().toString()); - ParsedDocument doc = mapper.parse( new SourceToParse( "test", "1", - BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("my_unsigned_long").endObject()), + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("field").endObject()), XContentType.JSON ) ); - IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(2, fields.length); IndexableField pointField = fields[0]; assertEquals(9223372036854775807L, pointField.numericValue().longValue()); @@ -328,27 +218,13 @@ public void testNullValue() throws IOException { public void testIgnoreMalformed() throws Exception { // test ignore_malformed is false by default { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .endObject() - .endObject() - .endObject() - .endObject() - ); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); - assertEquals(mapping, mapper.mappingSource().toString()); - + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); Object malformedValue1 = "a"; ThrowingRunnable runnable = () -> mapper.parse( new SourceToParse( "test", "_doc", - BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue1).endObject()), + BytesReference.bytes(jsonBuilder().startObject().field("field", malformedValue1).endObject()), XContentType.JSON ) ); @@ -360,7 +236,7 @@ public void testIgnoreMalformed() throws Exception { new SourceToParse( "test", "_doc", - BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue2).endObject()), + BytesReference.bytes(jsonBuilder().startObject().field("field", malformedValue2).endObject()), XContentType.JSON ) ); @@ -370,75 +246,64 @@ public void testIgnoreMalformed() throws Exception { // test ignore_malformed when set to true ignored malformed documents { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .field("ignore_malformed", true) - .endObject() - .endObject() - .endObject() - .endObject() + DocumentMapper mapper = createDocumentMapper( + fieldMapping(b -> b.field("type", "unsigned_long").field("ignore_malformed", true)) ); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); - assertEquals(mapping, mapper.mappingSource().toString()); - Object malformedValue1 = "a"; ParsedDocument doc = mapper.parse( new SourceToParse( "test", "1", - BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue1).endObject()), + BytesReference.bytes(jsonBuilder().startObject().field("field", malformedValue1).endObject()), XContentType.JSON ) ); - IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); - assertArrayEquals(new String[] { "my_unsigned_long" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored"))); + assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored"))); Object malformedValue2 = Boolean.FALSE; ParsedDocument doc2 = mapper.parse( new SourceToParse( "test", "1", - BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue2).endObject()), + BytesReference.bytes(jsonBuilder().startObject().field("field", malformedValue2).endObject()), XContentType.JSON ) ); - IndexableField[] fields2 = doc2.rootDoc().getFields("my_unsigned_long"); + IndexableField[] fields2 = doc2.rootDoc().getFields("field"); assertEquals(0, fields2.length); - assertArrayEquals(new String[] { "my_unsigned_long" }, TermVectorsService.getValues(doc2.rootDoc().getFields("_ignored"))); + assertArrayEquals(new String[] { "field" }, TermVectorsService.getValues(doc2.rootDoc().getFields("_ignored"))); } } public void testIndexingOutOfRangeValues() throws Exception { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("my_unsigned_long") - .field("type", "unsigned_long") - .endObject() - .endObject() - .endObject() - .endObject() - ); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); - + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); for (Object outOfRangeValue : new Object[] { "-1", -1L, "18446744073709551616", new BigInteger("18446744073709551616") }) { ThrowingRunnable runnable = () -> mapper.parse( new SourceToParse( "test", "_doc", - BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", outOfRangeValue).endObject()), + BytesReference.bytes(jsonBuilder().startObject().field("field", outOfRangeValue).endObject()), XContentType.JSON ) ); expectThrows(MapperParsingException.class, runnable); } } + + public void testFetchSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + UnsignedLongFieldMapper mapper = new UnsignedLongFieldMapper.Builder("field", settings).build(context); + assertEquals(List.of(0L), fetchSourceValue(mapper, 0L)); + assertEquals(List.of(9223372036854775807L), fetchSourceValue(mapper, 9223372036854775807L)); + assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(mapper, "18446744073709551615")); + assertEquals(List.of(), fetchSourceValue(mapper, "")); + + UnsignedLongFieldMapper nullValueMapper = new UnsignedLongFieldMapper.Builder("field", settings).nullValue("18446744073709551615") + .build(context); + assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, "")); + } } From 07470b5006ed3aac8a635a2953424f635ccd46c0 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Thu, 10 Sep 2020 09:55:18 -0400 Subject: [PATCH 08/13] Small edits in documentation --- docs/reference/mapping/types/unsigned_long.asciidoc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/reference/mapping/types/unsigned_long.asciidoc b/docs/reference/mapping/types/unsigned_long.asciidoc index 75debcd34f01c..9cc1e91790cd8 100644 --- a/docs/reference/mapping/types/unsigned_long.asciidoc +++ b/docs/reference/mapping/types/unsigned_long.asciidoc @@ -80,11 +80,12 @@ GET /my_index/_search For queries with sort on an `unsigned_long` field, -for a particular document {es} returns a sort value of the type `Long` +for a particular document {es} returns a sort value of the type `long` if the value of this document is within the range of long values, or of the type `BigInteger` if the value exceeds this range. -WARNING: Not all {es} clients can properly handle big integer values. +NOTE: Rest clients need to be able to handle big integer values +in json to support this field type correctly. [source,console] -------------------------------- @@ -99,12 +100,12 @@ GET /my_index/_search //TEST[continued] Similarly to sort values, script values of an `unsigned_long` field -produce `BigInteger` or `Long` values. The same values: `BigInteger` or -`Long` are used for `terms` aggregation. +return a number representing a `long` or `BigInteger`. +The same values: `long` or `BigInteger` are used for `terms` aggregations. ==== Queries with mixed numeric types -Search queries across several numeric types one of which `unsigned_long` are +Search queries across several numeric types one of which is `unsigned_long` are supported, except queries with sort. Thus, a sort query across two indexes where the same field name has an `unsigned_long` type in one index, and `long` type in another, doesn't produce correct results and must From 17912bc0549aaa275099513383517517c02dc7f9 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 16 Sep 2020 10:25:35 -0400 Subject: [PATCH 09/13] Address Julie's comment on documentation --- docs/reference/mapping/types/unsigned_long.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/mapping/types/unsigned_long.asciidoc b/docs/reference/mapping/types/unsigned_long.asciidoc index 9cc1e91790cd8..0f036a00fb3c6 100644 --- a/docs/reference/mapping/types/unsigned_long.asciidoc +++ b/docs/reference/mapping/types/unsigned_long.asciidoc @@ -84,8 +84,8 @@ for a particular document {es} returns a sort value of the type `long` if the value of this document is within the range of long values, or of the type `BigInteger` if the value exceeds this range. -NOTE: Rest clients need to be able to handle big integer values -in json to support this field type correctly. +NOTE: REST clients need to be able to handle big integer values +in JSON to support this field type correctly. [source,console] -------------------------------- @@ -100,12 +100,12 @@ GET /my_index/_search //TEST[continued] Similarly to sort values, script values of an `unsigned_long` field -return a number representing a `long` or `BigInteger`. -The same values: `long` or `BigInteger` are used for `terms` aggregations. +return a `Number` representing a `Long` or `BigInteger`. +The same values: `Long` or `BigInteger` are used for `terms` aggregations. ==== Queries with mixed numeric types -Search queries across several numeric types one of which is `unsigned_long` are +Searches with mixed numeric types one of which is `unsigned_long` are supported, except queries with sort. Thus, a sort query across two indexes where the same field name has an `unsigned_long` type in one index, and `long` type in another, doesn't produce correct results and must From b2eef4c4ca9997dddface371cfad3b1671fd438b Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 16 Sep 2020 15:36:09 -0400 Subject: [PATCH 10/13] Add check that unsigned_long field type can't be sorted with other types --- .../action/search/SearchPhaseController.java | 31 +++++++++++++++++++ .../xpack/unsignedlong/UnsignedLongTests.java | 28 ++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index a612e09a549f9..3bdb53d092947 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -426,6 +426,7 @@ ReducedQueryPhase reducedQueryPhase(Collection quer if (queryResults.isEmpty()) { throw new IllegalStateException(errorMsg); } + checkSameUnsignedLongSortFormat(queryResults); final QuerySearchResult firstResult = queryResults.stream().findFirst().get().queryResult(); final boolean hasSuggest = firstResult.suggest() != null; final boolean hasProfileResults = firstResult.hasProfileResults(); @@ -485,6 +486,36 @@ private static InternalAggregations reduceAggs(InternalAggregation.ReduceContext performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction()); } + /** + * Checks that query results from all shards have consistent unsigned_long format. + * Sort queries on a field that has long type in one index, and unsigned_long in another index + * don't work correctly. Throw an error if this kind of sorting is detected. + * //TODO: instead of throwing error, find a way to sort long and unsigned_long together + */ + private static void checkSameUnsignedLongSortFormat(Collection queryResults) { + boolean[] ulFormats = null; + boolean firstResult = true; + for (SearchPhaseResult entry : queryResults) { + DocValueFormat[] formats = entry.queryResult().sortValueFormats(); + if (formats == null) return; + if (firstResult) { + firstResult = false; + ulFormats = new boolean[formats.length]; + for (int i = 0; i < formats.length; i++) { + ulFormats[i] = formats[i] == DocValueFormat.UNSIGNED_LONG_SHIFTED ? true : false; + } + } else { + for (int i = 0; i < formats.length; i++) { + // if the format is unsigned_long in one shard, and something different in another shard + if (ulFormats[i] ^ (formats[i] == DocValueFormat.UNSIGNED_LONG_SHIFTED)) { + throw new IllegalArgumentException("Can't do sort across indices, as a field has [unsigned_long] type " + + "in one index, and different type in another index!"); + } + } + } + } + } + /* * Returns the size of the requested top documents (from + size) */ diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java index 213cf64751329..71631eccc04e1 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java @@ -7,6 +7,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; @@ -20,6 +21,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; +import java.io.IOException; import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; @@ -58,7 +60,7 @@ protected Collection> nodePlugins() { @Override public void setupSuiteScopeCluster() throws Exception { - Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", 3); + Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", 1); prepareCreate("idx").setMapping("ul_field", "type=unsigned_long").setSettings(settings).get(); List builders = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { @@ -264,4 +266,28 @@ public void testAggs() { assertEquals(expectedSum, sum.getValue(), 0.001); } } + + public void testSortDifferentFormatsShouldFail() throws IOException, InterruptedException { + Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", 1); + prepareCreate("idx2").setMapping("ul_field", "type=long").setSettings(settings).get(); + List builders = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + builders.add(client().prepareIndex("idx2").setSource(jsonBuilder().startObject().field("ul_field", values[i]).endObject())); + } + indexRandom(true, builders); + ensureSearchable(); + + Exception exception = expectThrows( + SearchPhaseExecutionException.class, + () -> client().prepareSearch() + .setIndices("idx", "idx2") + .setQuery(QueryBuilders.matchAllQuery()) + .addSort("ul_field", SortOrder.ASC) + .get() + ); + assertEquals( + exception.getCause().getMessage(), + "Can't do sort across indices, as a field has [unsigned_long] type in one index, and different type in another index!" + ); + } } From 7652e6671ce385c670231cc15292500311a8a785 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 16 Sep 2020 16:48:49 -0400 Subject: [PATCH 11/13] Fix build and test failures --- x-pack/plugin/mapper-unsigned-long/build.gradle | 2 -- .../unsignedlong/UnsignedLongFieldMapper.java | 2 +- .../UnsignedLongFieldMapperTests.java | 2 +- .../xpack/unsignedlong/UnsignedLongTests.java | 16 ++++++++-------- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/build.gradle b/x-pack/plugin/mapper-unsigned-long/build.gradle index e0eaca675756a..f7aba6993a974 100644 --- a/x-pack/plugin/mapper-unsigned-long/build.gradle +++ b/x-pack/plugin/mapper-unsigned-long/build.gradle @@ -21,5 +21,3 @@ dependencies { compileOnly project(path: xpackModule('core'), configuration: 'default') testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') } - -integTest.enabled = false diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 8ae29396f2cce..c2b6a16223d82 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -451,7 +451,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 6905b90d03bc5..01ab082c6c0bd 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -292,7 +292,7 @@ public void testIndexingOutOfRangeValues() throws Exception { } } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java index 71631eccc04e1..19c9f80b081cd 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java @@ -73,7 +73,7 @@ public void setupSuiteScopeCluster() throws Exception { public void testSort() { // asc sort { - SearchResponse response = client().prepareSearch() + SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .setSize(numDocs) .addSort("ul_field", SortOrder.ASC) @@ -88,7 +88,7 @@ public void testSort() { } // desc sort { - SearchResponse response = client().prepareSearch() + SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .setSize(numDocs) .addSort("ul_field", SortOrder.DESC) @@ -103,7 +103,7 @@ public void testSort() { } // asc sort with search_after as Long { - SearchResponse response = client().prepareSearch() + SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .setSize(numDocs) .addSort("ul_field", SortOrder.ASC) @@ -119,7 +119,7 @@ public void testSort() { } // asc sort with search_after as BigInteger { - SearchResponse response = client().prepareSearch() + SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .setSize(numDocs) .addSort("ul_field", SortOrder.ASC) @@ -135,7 +135,7 @@ public void testSort() { } // asc sort with search_after as BigInteger in String format { - SearchResponse response = client().prepareSearch() + SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .setSize(numDocs) .addSort("ul_field", SortOrder.ASC) @@ -151,7 +151,7 @@ public void testSort() { } // asc sort with search_after of negative value should fail { - SearchRequestBuilder srb = client().prepareSearch() + SearchRequestBuilder srb = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .setSize(numDocs) .addSort("ul_field", SortOrder.ASC) @@ -161,7 +161,7 @@ public void testSort() { } // asc sort with search_after of value>=2^64 should fail { - SearchRequestBuilder srb = client().prepareSearch() + SearchRequestBuilder srb = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .setSize(numDocs) .addSort("ul_field", SortOrder.ASC) @@ -171,7 +171,7 @@ public void testSort() { } // desc sort with search_after as BigInteger { - SearchResponse response = client().prepareSearch() + SearchResponse response = client().prepareSearch("idx") .setQuery(QueryBuilders.matchAllQuery()) .setSize(numDocs) .addSort("ul_field", SortOrder.DESC) From 24cbe55162a29463c22de9815187b7e48fc65844 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 23 Sep 2020 10:37:23 -0400 Subject: [PATCH 12/13] Rename the method for validating consistency of merge formats --- .../elasticsearch/action/search/SearchPhaseController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 3bdb53d092947..92c4ff0bdcf2b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -426,7 +426,7 @@ ReducedQueryPhase reducedQueryPhase(Collection quer if (queryResults.isEmpty()) { throw new IllegalStateException(errorMsg); } - checkSameUnsignedLongSortFormat(queryResults); + validateMergeSortValueFormats(queryResults); final QuerySearchResult firstResult = queryResults.stream().findFirst().get().queryResult(); final boolean hasSuggest = firstResult.suggest() != null; final boolean hasProfileResults = firstResult.hasProfileResults(); @@ -492,7 +492,7 @@ private static InternalAggregations reduceAggs(InternalAggregation.ReduceContext * don't work correctly. Throw an error if this kind of sorting is detected. * //TODO: instead of throwing error, find a way to sort long and unsigned_long together */ - private static void checkSameUnsignedLongSortFormat(Collection queryResults) { + private static void validateMergeSortValueFormats(Collection queryResults) { boolean[] ulFormats = null; boolean firstResult = true; for (SearchPhaseResult entry : queryResults) { From cca6b30e1d47295497fc1a34d2ca94881a62317d Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 23 Sep 2020 11:10:52 -0400 Subject: [PATCH 13/13] Change unsigned_long mapper based on recent master changes --- .../unsignedlong/UnsignedLongFieldMapper.java | 36 ++++--------------- .../UnsignedLongFieldMapperTests.java | 16 +++++++++ .../UnsignedLongFieldTypeTests.java | 2 +- 3 files changed, 23 insertions(+), 31 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index c2b6a16223d82..9138aa1d0d45a 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -10,14 +10,10 @@ import com.fasterxml.jackson.core.exc.InputCoercionException; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.settings.Settings; @@ -26,7 +22,6 @@ import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; @@ -117,6 +112,7 @@ public UnsignedLongFieldMapper build(BuilderContext context) { UnsignedLongFieldType fieldType = new UnsignedLongFieldType( buildFullName(context), indexed.getValue(), + stored.getValue(), hasDocValues.getValue(), meta.getValue() ); @@ -128,12 +124,12 @@ public UnsignedLongFieldMapper build(BuilderContext context) { public static final class UnsignedLongFieldType extends SimpleMappedFieldType { - public UnsignedLongFieldType(String name, boolean indexed, boolean hasDocValues, Map meta) { - super(name, indexed, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + public UnsignedLongFieldType(String name, boolean indexed, boolean isStored, boolean hasDocValues, Map meta) { + super(name, indexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); } public UnsignedLongFieldType(String name) { - this(name, true, true, Collections.emptyMap()); + this(name, true, false, true, Collections.emptyMap()); } @Override @@ -141,15 +137,6 @@ public String typeName() { return CONTENT_TYPE; } - @Override - public Query existsQuery(QueryShardContext context) { - if (hasDocValues()) { - return new DocValuesFieldExistsQuery(name()); - } else { - return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); - } - } - @Override public Query termQuery(Object value, QueryShardContext context) { failIfNotIndexed(); @@ -157,11 +144,7 @@ public Query termQuery(Object value, QueryShardContext context) { if (longValue == null) { return new MatchNoDocsQuery(); } - Query query = LongPoint.newExactQuery(name(), unsignedToSortableSignedLong(longValue)); - if (boost() != 1f) { - query = new BoostQuery(query, boost()); - } - return query; + return LongPoint.newExactQuery(name(), unsignedToSortableSignedLong(longValue)); } @Override @@ -182,11 +165,7 @@ public Query termsQuery(List values, QueryShardContext context) { if (upTo != lvalues.length) { lvalues = Arrays.copyOf(lvalues, upTo); } - Query query = LongPoint.newSetQuery(name(), lvalues); - if (boost() != 1f) { - query = new BoostQuery(query, boost()); - } - return query; + return LongPoint.newSetQuery(name(), lvalues); } @Override @@ -214,9 +193,6 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); } } - if (boost() != 1f) { - query = new BoostQuery(query, boost()); - } return query; } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 01ab082c6c0bd..089f04d414668 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; @@ -47,6 +48,11 @@ protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "unsigned_long"); } + @Override + protected void writeFieldValue(XContentBuilder builder) throws IOException { + builder.value(123); + } + public void testDefaults() throws Exception { XContentBuilder mapping = fieldMapping(b -> b.field("type", "unsigned_long")); DocumentMapper mapper = createDocumentMapper(mapping); @@ -306,4 +312,14 @@ public void testFetchSourceValue() throws IOException { .build(context); assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, "")); } + + public void testExistsQueryDocValuesDisabled() throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> { + minimalMapping(b); + b.field("doc_values", false); + })); + assertExistsQuery(mapperService); + assertParseMinimalWarnings(); + } + } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java index e3e881c5389c6..83fbcc98fdd68 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java @@ -47,7 +47,7 @@ public void testTermsQuery() { } public void testRangeQuery() { - UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long", true, false, null); + UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long", true, false, false, null); assertEquals( LongPoint.newRangeQuery("my_unsigned_long", -9223372036854775808L, -9223372036854775808L),